From f6ee7f9f030ddedc0c0783bc788869aef85a6122 Mon Sep 17 00:00:00 2001 From: Denis 'GNUtoo' Carikli Date: Tue, 24 Oct 2023 17:37:41 +0200 Subject: [PATCH] website-build: cache tarball builds. Before we always rebuilt everything to make sure to avoid situations where newer source files weren't added in the builds. However this is quite costly as for instance 'make check' ends up rebuilding the tarball even after running 'make' right before. Since the Guix revision is fixed and that the build shell script (build.sh) also hardcodes the revision being used to build the tools, we can rely on that to only rebuild if they changed. For external repositories (that are used with ./configure --with--path=PATH [...]) we use a hash of the timestamps of all the files in these directories. While it's not optimal as any unrelated change or new file would trigger a rebuild, it at least improves the situation when people do not use external repositories Signed-off-by: Denis 'GNUtoo' Carikli --- website-build/.gitignore | 5 +++ website-build/Makefile.am | 94 ++++++++++++++++++++++++++++++++++++--- website-build/build.sh | 6 ++- 3 files changed, 96 insertions(+), 9 deletions(-) diff --git a/website-build/.gitignore b/website-build/.gitignore index 68bc85b..801142e 100644 --- a/website-build/.gitignore +++ b/website-build/.gitignore @@ -4,10 +4,15 @@ /config.log /config.status /configure +/guix-commit.txt /id_oauth2_bearer /install-sh +/lbwww-img-path-sha1sum.txt +/lbwww-path-sha1sum.txt /lighttpd.conf /Makefile /Makefile.in /missing +/untitled-path-sha1sum.txt +/website-commits.txt /website.tar.gz \ No newline at end of file diff --git a/website-build/Makefile.am b/website-build/Makefile.am index 50426f3..f59202b 100644 --- a/website-build/Makefile.am +++ b/website-build/Makefile.am @@ -12,9 +12,11 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see . +GUIX_COMMIT ?= 07f19ef04b5a8f4d7a12a8940333e67db8da81c0 + DOMAIN := gnu.org/software/gnuboot -.PHONY: all build check help deploy website.tar.gz +.PHONY: all check help deploy all: website.tar.gz @@ -31,10 +33,86 @@ BUILD_OPTIONS += --with-untitled-path $(UNTITLED_PATH) GUIX_SHARE_OPTIONS += --share=`realpath $(UNTITLED_PATH)` endif +# We want to update the tarball when guix commit changes. The trick to +# make that work is to only create or update a file when the revision +# changes. if WANT_GUIX -build: +guix-commit.txt: Makefile + if [ ! -f $@ ] || [ "$(cat $@)" != "$(GUIX_COMMIT)" ] ; then \ + echo -n "$(GUIX_COMMIT)" > $@ ; \ + fi +else +guix-commit.txt: Makefile + if [ ! -f $@ ] || [ "$(cat $@)" != "" ] ; then \ + echo -n "" > $@ ; \ + fi +endif + +# When we use an external lbwww repository, we don't want to rebuild +# the tarball each time in case something changes. Since Make uses +# timestamps, we store the hash of the timestamps and compare that to +# the hash of the last build to see if we need to rebuild the tarball. +if WANT_LBWWW_PATH +LBWWW_PATH_HASH := `ls -alR --full-time $(LBWWW_PATH) | sha1sum` +lbwww-path-sha1sum.txt: Makefile + if [ ! -f $@ ] || [ "$(cat $@)" != "$(LBWWW_PATH_HASH)" ] ; then \ + echo -n "$(LBWWW_PATH_HASH)" > $@ ; \ + fi +else +lbwww-path-sha1sum.txt: Makefile + if [ ! -f $@ ] || [ "$(cat $@)" != "" ] ; then \ + echo -n "" > $@ ; \ + fi +endif + +# When we use an external lbwww-img repository, we don't want to +# rebuild the tarball each time in case something changes. Since Make +# uses timestamps, we store the hash of the timestamps and compare +# that to the hash of the last build to see if we need to rebuild the +# tarball. +if WANT_LBWWW_IMG_PATH +LBWWW_IMG_PATH_HASH := `ls -alR --full-time $(LBWWW_IMG_PATH) | sha1sum` +lbwww-img-path-sha1sum.txt: Makefile + if [ ! -f $@ ] || [ "$(cat $@)" != "$(LBWWW_IMG_PATH_HASH)" ] ; then \ + echo -n "$(LBWWW_PATH_HASH)" > $@ ; \ + fi +else +lbwww-img-path-sha1sum.txt: Makefile + if [ ! -f $@ ] || [ "$(cat $@)" != "" ] ; then \ + echo -n "" > $@ ; \ + fi +endif + +# When we use an external untitled repository, we don't want to +# rebuild the tarball each time in case something changes. Since Make +# uses timestamps, we store the hash of the timestamps and compare +# that to the hash of the last build to see if we need to rebuild the +# tarball. +if WANT_UNTITLED_PATH +UNTITLED_PATH_HASH := `ls -alR --full-time $(UNTITLED_PATH) | sha1sum` +untitled-path-sha1sum.txt: Makefile + if [ ! -f $@ ] || [ "$(cat $@)" != "$(UNTITLED_PATH_HASH)" ] ; then \ + echo -n "$(UNTITLED_PATH_HASH)" > $@ ; \ + fi +else +untitled-path-sha1sum.txt: Makefile + if [ ! -f $@ ] || [ "$(cat $@)" != "" ] ; then \ + echo -n "" > $@ ; \ + fi +endif + +WEBSITE_BUILD_DEPENDENCIES = \ + build.sh \ + guix-commit.txt \ + lbwww-path-sha1sum.txt \ + lbwww-img-path-sha1sum.txt \ + Makefile \ + untitled-path-sha1sum.txt + +if WANT_GUIX +website-commits.txt: $(WEBSITE_BUILD_DEPENDENCIES) guix time-machine \ - --commit=07f19ef04b5a8f4d7a12a8940333e67db8da81c0 \ + --commit=$(GUIX_COMMIT) \ -- shell \ --container \ --network \ @@ -50,9 +128,11 @@ build: sed \ -- \ ./build.sh $(BUILD_OPTIONS) + cat $(WEBSITE_BUILD_DEPENDENCIES) > $@ else -build: - ./build.sh $(BUILD_OPTIONS) +website-commits.txt: $(WEBSITE_BUILD_DEPENDENCIES) + ./build.sh $(BUILD_OPTIONS) + cat $(WEBSITE_BUILD_DEPENDENCIES) > $@ endif if WANT_GUIX @@ -110,7 +190,7 @@ endif endif RSYNC_OPTIONS := -av --progress -deploy: build +deploy: website.tar.gz rm -rf deploy mkdir -p deploy tar xf website.tar.gz -C deploy @@ -119,7 +199,7 @@ deploy: build $(RSYNC_DESTINATION)/ # See https://reproducible-builds.org/docs/archives/ for more details -website.tar.gz: build +website.tar.gz: website-commits.txt tar \ --exclude-vcs \ --format=gnu \ diff --git a/website-build/build.sh b/website-build/build.sh index bf71935..56b04ba 100755 --- a/website-build/build.sh +++ b/website-build/build.sh @@ -19,9 +19,11 @@ EX_USAGE=64 lbwww_uri="https://git.savannah.gnu.org/git/gnuboot.git" lbwww_path="" +lbwww_commit="4383810adf1b01c408c266e87d526282cf8cd635" lbwww_img_uri="https://git.savannah.gnu.org/git/gnuboot.git" lbwww_img_path="" +lbwww_img_commit="4383810adf1b01c408c266e87d526282cf8cd635" untitled_uri="https://notabug.org/untitled/untitled.git" untitled_path="" @@ -134,9 +136,9 @@ set -e sync_repo "untitled" \ "${untitled_uri}" "${untitled_path}" "${untitled_commit}" sync_repo "untitled/www/lbwww" \ - "${lbwww_uri}" "${lbwww_path}" "origin/main" + "${lbwww_uri}" "${lbwww_path}" "${lbwww_commit}" sync_repo "untitled/www/lbwww/site/img" \ - "${lbwww_img_uri}" "${lbwww_img_path}" "origin/main" + "${lbwww_img_uri}" "${lbwww_img_path}" "${lbwww_img_commit}" cd untitled ./build sites lbwww