Message ID | 20180331042228.7432-1-champetier.etienne@gmail.com |
---|---|
State | Changes Requested |
Delegated to: | John Crispin |
Headers | show |
Series | [LEDE-DEV] build: log time taken by each packages/steps | expand |
Hi Etienne, On 03/30/2018 09:22 PM, Etienne Champetier wrote: > The idea is to easily get the list of packages taking the most time to build, > and see if we can improve them I like the idea a lot although I was not getting timestamps printed for all steps of the build, for instance tools/* seems to have been completely missing while I would expect to see some timestamps printed there to possibly optimize the tools builds. This also applied to the toolchain part, does that match what you would be expecting? > > Use SetupHostCommand as recommended by John > > Signed-off-by: Etienne Champetier <champetier.etienne@gmail.com> > --- > include/prereq-build.mk | 5 +++++ > include/subdir.mk | 1 + > 2 files changed, 6 insertions(+) > > diff --git a/include/prereq-build.mk b/include/prlereq-build.mk > index 6a423d2c7d..7d5b2f2b45 100644 > --- a/include/prereq-build.mk > +++ b/include/prereq-build.mk > @@ -139,6 +139,11 @@ $(eval $(call SetupHostCommand,bzip2,Please install 'bzip2', \ > $(eval $(call SetupHostCommand,wget,Please install GNU 'wget', \ > wget --version | grep GNU)) > > +$(eval $(call SetupHostCommand,gtime,Please install GNU 'time', \ > + gtime --version 2>&1 | grep GNU, \ > + /usr/bin/time --version 2>&1 | grep GNU, \ > + time --version 2>&1 | grep GNU)) > + > $(eval $(call SetupHostCommand,perl,Please install Perl 5.x, \ > perl --version | grep "perl.*v5")) > > diff --git a/include/subdir.mk b/include/subdir.mk > index 79a80528ae..546ed57ae6 100644 > --- a/include/subdir.mk > +++ b/include/subdir.mk > @@ -43,6 +43,7 @@ log_make = \ > $(if $(BUILD_LOG), \ > set -o pipefail; \ > mkdir -p $(BUILD_LOG_DIR)/$(1)$(if $(4),/$(4));) \ > + gtime -f "time:$(1)$(if $(4),/$(4))/$(if $(3),$(3)-)$(2)\#%U\#%S\#%e" -- \ Can we add a space between time and $(1) here? > $$(SUBMAKE) $(subdir_make_opts) $(if $(3),$(3)-)$(2) \ > $(if $(BUILD_LOG),SILENT= 2>&1 | tee $(BUILD_LOG_DIR)/$(1)$(if $(4),/$(4))/$(if $(3),$(3)-)$(2).txt) > >
Hi Florian, 2018-04-04 13:44 GMT-03:00 Florian Fainelli <f.fainelli@gmail.com>: > Hi Etienne, > > On 03/30/2018 09:22 PM, Etienne Champetier wrote: >> The idea is to easily get the list of packages taking the most time to build, >> and see if we can improve them > > I like the idea a lot although I was not getting timestamps printed for > all steps of the build, for instance tools/* seems to have been > completely missing while I would expect to see some timestamps printed > there to possibly optimize the tools builds. This also applied to the > toolchain part, does that match what you would be expecting? I just wanted to know what packages take the most times but I agree that extending it to all build step is a good thing. I don't fully understand the build process, so if you see a better spot to put this please go ahead > >> >> Use SetupHostCommand as recommended by John >> >> Signed-off-by: Etienne Champetier <champetier.etienne@gmail.com> >> --- >> include/prereq-build.mk | 5 +++++ >> include/subdir.mk | 1 + >> 2 files changed, 6 insertions(+) >> >> diff --git a/include/prereq-build.mk b/include/prlereq-build.mk >> index 6a423d2c7d..7d5b2f2b45 100644 >> --- a/include/prereq-build.mk >> +++ b/include/prereq-build.mk >> @@ -139,6 +139,11 @@ $(eval $(call SetupHostCommand,bzip2,Please install 'bzip2', \ >> $(eval $(call SetupHostCommand,wget,Please install GNU 'wget', \ >> wget --version | grep GNU)) >> >> +$(eval $(call SetupHostCommand,gtime,Please install GNU 'time', \ >> + gtime --version 2>&1 | grep GNU, \ >> + /usr/bin/time --version 2>&1 | grep GNU, \ >> + time --version 2>&1 | grep GNU)) >> + >> $(eval $(call SetupHostCommand,perl,Please install Perl 5.x, \ >> perl --version | grep "perl.*v5")) >> >> diff --git a/include/subdir.mk b/include/subdir.mk >> index 79a80528ae..546ed57ae6 100644 >> --- a/include/subdir.mk >> +++ b/include/subdir.mk >> @@ -43,6 +43,7 @@ log_make = \ >> $(if $(BUILD_LOG), \ >> set -o pipefail; \ >> mkdir -p $(BUILD_LOG_DIR)/$(1)$(if $(4),/$(4));) \ >> + gtime -f "time:$(1)$(if $(4),/$(4))/$(if $(3),$(3)-)$(2)\#%U\#%S\#%e" -- \ > > Can we add a space between time and $(1) here? Fine with me (I just want something easy to extract then parse) Etienne > >> $$(SUBMAKE) $(subdir_make_opts) $(if $(3),$(3)-)$(2) \ >> $(if $(BUILD_LOG),SILENT= 2>&1 | tee $(BUILD_LOG_DIR)/$(1)$(if $(4),/$(4))/$(if $(3),$(3)-)$(2).txt) >> >> > > -- > Florian
On 31/03/18 06:22, Etienne Champetier wrote: > The idea is to easily get the list of packages taking the most time to build, > and see if we can improve them > > Use SetupHostCommand as recommended by John > > Signed-off-by: Etienne Champetier <champetier.etienne@gmail.com> > --- > include/prereq-build.mk | 5 +++++ > include/subdir.mk | 1 + > 2 files changed, 6 insertions(+) > > diff --git a/include/prereq-build.mk b/include/prereq-build.mk > index 6a423d2c7d..7d5b2f2b45 100644 > --- a/include/prereq-build.mk > +++ b/include/prereq-build.mk > @@ -139,6 +139,11 @@ $(eval $(call SetupHostCommand,bzip2,Please install 'bzip2', \ > $(eval $(call SetupHostCommand,wget,Please install GNU 'wget', \ > wget --version | grep GNU)) > > +$(eval $(call SetupHostCommand,gtime,Please install GNU 'time', \ > + gtime --version 2>&1 | grep GNU, \ > + /usr/bin/time --version 2>&1 | grep GNU, \ Hi, i think the line above should be removed. happy to merge the patch once that has been done and florian's space request was added John > + time --version 2>&1 | grep GNU)) > + > $(eval $(call SetupHostCommand,perl,Please install Perl 5.x, \ > perl --version | grep "perl.*v5")) > > diff --git a/include/subdir.mk b/include/subdir.mk > index 79a80528ae..546ed57ae6 100644 > --- a/include/subdir.mk > +++ b/include/subdir.mk > @@ -43,6 +43,7 @@ log_make = \ > $(if $(BUILD_LOG), \ > set -o pipefail; \ > mkdir -p $(BUILD_LOG_DIR)/$(1)$(if $(4),/$(4));) \ > + gtime -f "time:$(1)$(if $(4),/$(4))/$(if $(3),$(3)-)$(2)\#%U\#%S\#%e" -- \ > $$(SUBMAKE) $(subdir_make_opts) $(if $(3),$(3)-)$(2) \ > $(if $(BUILD_LOG),SILENT= 2>&1 | tee $(BUILD_LOG_DIR)/$(1)$(if $(4),/$(4))/$(if $(3),$(3)-)$(2).txt) >
Hi Florian, 2018-04-04 9:44 GMT-07:00 Florian Fainelli <f.fainelli@gmail.com>: > Hi Etienne, > > On 03/30/2018 09:22 PM, Etienne Champetier wrote: >> The idea is to easily get the list of packages taking the most time to build, >> and see if we can improve them > > I like the idea a lot although I was not getting timestamps printed for > all steps of the build, for instance tools/* seems to have been > completely missing while I would expect to see some timestamps printed > there to possibly optimize the tools builds. This also applied to the > toolchain part, does that match what you would be expecting? just ran a build after "make dirclean" $ make -j4 V=99 2>&1 | grep '^time: ' time: target/linux/prereq#0.27#0.04#0.31 time: tools/flock/compile#0.18#0.08#0.49 time: tools/sed/compile#17.93#10.52#24.20 time: tools/xz/compile#29.36#8.00#25.35 ... And a second one: $ make -j4 V=99 2>&1 | grep '^time: ' time: target/linux/compile#5.24#3.41#5.32 time: package/libs/ncurses/host-compile#0.13#0.10#0.35 time: package/libs/toolchain/compile#0.11#0.13#0.45 ... So it's just that tools/* is skipped when it's already built > >> >> Use SetupHostCommand as recommended by John >> >> Signed-off-by: Etienne Champetier <champetier.etienne@gmail.com> >> --- >> include/prereq-build.mk | 5 +++++ >> include/subdir.mk | 1 + >> 2 files changed, 6 insertions(+) >> >> diff --git a/include/prereq-build.mk b/include/prlereq-build.mk >> index 6a423d2c7d..7d5b2f2b45 100644 >> --- a/include/prereq-build.mk >> +++ b/include/prereq-build.mk >> @@ -139,6 +139,11 @@ $(eval $(call SetupHostCommand,bzip2,Please install 'bzip2', \ >> $(eval $(call SetupHostCommand,wget,Please install GNU 'wget', \ >> wget --version | grep GNU)) >> >> +$(eval $(call SetupHostCommand,gtime,Please install GNU 'time', \ >> + gtime --version 2>&1 | grep GNU, \ >> + /usr/bin/time --version 2>&1 | grep GNU, \ >> + time --version 2>&1 | grep GNU)) >> + >> $(eval $(call SetupHostCommand,perl,Please install Perl 5.x, \ >> perl --version | grep "perl.*v5")) >> >> diff --git a/include/subdir.mk b/include/subdir.mk >> index 79a80528ae..546ed57ae6 100644 >> --- a/include/subdir.mk >> +++ b/include/subdir.mk >> @@ -43,6 +43,7 @@ log_make = \ >> $(if $(BUILD_LOG), \ >> set -o pipefail; \ >> mkdir -p $(BUILD_LOG_DIR)/$(1)$(if $(4),/$(4));) \ >> + gtime -f "time:$(1)$(if $(4),/$(4))/$(if $(3),$(3)-)$(2)\#%U\#%S\#%e" -- \ > > Can we add a space between time and $(1) here? will do > >> $$(SUBMAKE) $(subdir_make_opts) $(if $(3),$(3)-)$(2) \ >> $(if $(BUILD_LOG),SILENT= 2>&1 | tee $(BUILD_LOG_DIR)/$(1)$(if $(4),/$(4))/$(if $(3),$(3)-)$(2).txt) >> >> > > -- > Florian
Hi John, 2018-04-29 23:41 GMT-07:00 John Crispin <john@phrozen.org>: > > > On 31/03/18 06:22, Etienne Champetier wrote: >> >> The idea is to easily get the list of packages taking the most time to >> build, >> and see if we can improve them >> >> Use SetupHostCommand as recommended by John >> >> Signed-off-by: Etienne Champetier <champetier.etienne@gmail.com> >> --- >> include/prereq-build.mk | 5 +++++ >> include/subdir.mk | 1 + >> 2 files changed, 6 insertions(+) >> >> diff --git a/include/prereq-build.mk b/include/prereq-build.mk >> index 6a423d2c7d..7d5b2f2b45 100644 >> --- a/include/prereq-build.mk >> +++ b/include/prereq-build.mk >> @@ -139,6 +139,11 @@ $(eval $(call SetupHostCommand,bzip2,Please install >> 'bzip2', \ >> $(eval $(call SetupHostCommand,wget,Please install GNU 'wget', \ >> wget --version | grep GNU)) >> +$(eval $(call SetupHostCommand,gtime,Please install GNU 'time', \ >> + gtime --version 2>&1 | grep GNU, \ >> + /usr/bin/time --version 2>&1 | grep GNU, \ > > > Hi, > i think the line above should be removed. happy to merge the patch once that > has been done and florian's space request was added You are right, SetupHostCommand find "time" binary without "/usr/bin/time" line sending v2 > John > > >> + time --version 2>&1 | grep GNU)) >> + >> $(eval $(call SetupHostCommand,perl,Please install Perl 5.x, \ >> perl --version | grep "perl.*v5")) >> diff --git a/include/subdir.mk b/include/subdir.mk >> index 79a80528ae..546ed57ae6 100644 >> --- a/include/subdir.mk >> +++ b/include/subdir.mk >> @@ -43,6 +43,7 @@ log_make = \ >> $(if $(BUILD_LOG), \ >> set -o pipefail; \ >> mkdir -p $(BUILD_LOG_DIR)/$(1)$(if $(4),/$(4));) \ >> + gtime -f "time:$(1)$(if $(4),/$(4))/$(if >> $(3),$(3)-)$(2)\#%U\#%S\#%e" -- \ >> $$(SUBMAKE) $(subdir_make_opts) $(if $(3),$(3)-)$(2) \ >> $(if $(BUILD_LOG),SILENT= 2>&1 | tee >> $(BUILD_LOG_DIR)/$(1)$(if $(4),/$(4))/$(if $(3),$(3)-)$(2).txt) >> > >
diff --git a/include/prereq-build.mk b/include/prereq-build.mk index 6a423d2c7d..7d5b2f2b45 100644 --- a/include/prereq-build.mk +++ b/include/prereq-build.mk @@ -139,6 +139,11 @@ $(eval $(call SetupHostCommand,bzip2,Please install 'bzip2', \ $(eval $(call SetupHostCommand,wget,Please install GNU 'wget', \ wget --version | grep GNU)) +$(eval $(call SetupHostCommand,gtime,Please install GNU 'time', \ + gtime --version 2>&1 | grep GNU, \ + /usr/bin/time --version 2>&1 | grep GNU, \ + time --version 2>&1 | grep GNU)) + $(eval $(call SetupHostCommand,perl,Please install Perl 5.x, \ perl --version | grep "perl.*v5")) diff --git a/include/subdir.mk b/include/subdir.mk index 79a80528ae..546ed57ae6 100644 --- a/include/subdir.mk +++ b/include/subdir.mk @@ -43,6 +43,7 @@ log_make = \ $(if $(BUILD_LOG), \ set -o pipefail; \ mkdir -p $(BUILD_LOG_DIR)/$(1)$(if $(4),/$(4));) \ + gtime -f "time:$(1)$(if $(4),/$(4))/$(if $(3),$(3)-)$(2)\#%U\#%S\#%e" -- \ $$(SUBMAKE) $(subdir_make_opts) $(if $(3),$(3)-)$(2) \ $(if $(BUILD_LOG),SILENT= 2>&1 | tee $(BUILD_LOG_DIR)/$(1)$(if $(4),/$(4))/$(if $(3),$(3)-)$(2).txt)
The idea is to easily get the list of packages taking the most time to build, and see if we can improve them Use SetupHostCommand as recommended by John Signed-off-by: Etienne Champetier <champetier.etienne@gmail.com> --- include/prereq-build.mk | 5 +++++ include/subdir.mk | 1 + 2 files changed, 6 insertions(+)