GNUBoot
/
coreboot-kgpe-d16
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

util/crossgcc: Add option to get packages from coreboot's mirror 

coreboot has been keeping a mirror of all the toolchain packages used
for releases for quite a while now.  This adds an option to fetch the
packages from the coreboot mirror directly to buildgcc.

This can help with both our releases and when one of the various
servers experiences interruptions or changes a path.

To do this, the URL and filename needed to be split apart, which led to
quite a few changes in the buildgcc script.

Signed-off-by: Martin Roth <gaumless@gmail.com>
Change-Id: I7df58dca152e7bfe9fde34d290e05b52515b20d9
Reviewed-on: https://review.coreboot.org/c/coreboot/+/70053
Tested-by: build bot (Jenkins) <no-reply@coreboot.org>
Reviewed-by: Elyes Haouas <ehaouas@noos.fr>
Reviewed-by: Felix Singer <felixsinger@posteo.net>
This commit is contained in:
Martin Roth 2022-11-27 12:55:31 -07:00 committed by Felix Singer
parent 50139d00bd
commit 3b32af950d
2 changed files with 63 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
util/crossgcc/Makefile.inc
View File

@ -3,6 +3,7 @@
TOOLCHAIN_ARCHES := i386 x64 arm aarch64 riscv ppc64 nds32le TOOLCHAIN_ARCHES := i386 x64 arm aarch64 riscv ppc64 nds32le
help_toolchain help:: help_toolchain help::
@echo
@echo '*** Toolchain targets ***' @echo '*** Toolchain targets ***'
@echo ' crossgcc - Build coreboot cross-compilers for all platforms' @echo ' crossgcc - Build coreboot cross-compilers for all platforms'
@echo ' crossgcc-clean - Remove all built coreboot cross-compilers' @echo ' crossgcc-clean - Remove all built coreboot cross-compilers'
@ -14,6 +15,7 @@ help_toolchain help::
@echo ' ARCH can be "$(subst $(spc),"$(comma) ",$(TOOLCHAIN_ARCHES))"' @echo ' ARCH can be "$(subst $(spc),"$(comma) ",$(TOOLCHAIN_ARCHES))"'
@echo ' Use "make [target] CPUS=#" to build toolchain using multiple cores' @echo ' Use "make [target] CPUS=#" to build toolchain using multiple cores'
@echo ' Use "make [target] DEST=some/path" to install toolchain there' @echo ' Use "make [target] DEST=some/path" to install toolchain there'
@echo ' Use "make [target] BUILDGCC_OPTIONS="-m" to get packages from coreboot mirror"
@echo @echo
# For the toolchain builds, use CPUS=x to use multiple processors to build # For the toolchain builds, use CPUS=x to use multiple processors to build

104
util/crossgcc/buildgcc
View File

@ -30,6 +30,8 @@ DESTDIR=
SAVETEMPS=0 SAVETEMPS=0
BOOTSTRAP=0 BOOTSTRAP=0
THREADS=1 THREADS=1
USE_COREBOOT_MIRROR=0
COREBOOT_MIRROR_URL="https://www.coreboot.org/releases/crossgcc-sources"
# GCC toolchain version numbers # GCC toolchain version numbers
GMP_VERSION=6.2.1 GMP_VERSION=6.2.1
@ -43,29 +45,46 @@ CLANG_VERSION=15.0.6
CMAKE_VERSION=3.25.0 CMAKE_VERSION=3.25.0
NASM_VERSION=2.15.05 NASM_VERSION=2.15.05
# GCC toolchain archive locations # Filename for each package
# These are sanitized by the jenkins toolchain test builder, so if GMP_ARCHIVE="gmp-${GMP_VERSION}.tar.xz"
MPFR_ARCHIVE="mpfr-${MPFR_VERSION}.tar.xz"
MPC_ARCHIVE="mpc-${MPC_VERSION}.tar.gz"
GCC_ARCHIVE="gcc-${GCC_VERSION}.tar.xz"
BINUTILS_ARCHIVE="binutils-${BINUTILS_VERSION}.tar.xz"
IASL_ARCHIVE="${IASL_VERSION}.tar.gz"
# CLANG toolchain FILE locations
LLVM_ARCHIVE="llvm-${CLANG_VERSION}.src.tar.xz"
CLANG_ARCHIVE="clang-${CLANG_VERSION}.src.tar.xz"
CRT_ARCHIVE="compiler-rt-${CLANG_VERSION}.src.tar.xz"
CTE_ARCHIVE="clang-tools-extra-${CLANG_VERSION}.src.tar.xz"
LLVMCMAKE_ARCHIVE="cmake-${CLANG_VERSION}.src.tar.xz"
CMAKE_ARCHIVE="cmake-${CMAKE_VERSION}.tar.gz"
NASM_ARCHIVE="nasm-${NASM_VERSION}.tar.bz2"
# These URLs are sanitized by the jenkins toolchain test builder, so if
# a completely new URL is added here, it probably needs to be added # a completely new URL is added here, it probably needs to be added
# to the jenkins build as well, or the builder won't download it. # to the jenkins build as well, or the builder won't download it.
GMP_ARCHIVE="https://ftpmirror.gnu.org/gmp/gmp-${GMP_VERSION}.tar.xz"
MPFR_ARCHIVE="https://ftpmirror.gnu.org/mpfr/mpfr-${MPFR_VERSION}.tar.xz"
MPC_ARCHIVE="https://ftpmirror.gnu.org/mpc/mpc-${MPC_VERSION}.tar.gz"
GCC_ARCHIVE="https://ftpmirror.gnu.org/gcc/gcc-${GCC_VERSION}/gcc-${GCC_VERSION}.tar.xz"
BINUTILS_ARCHIVE="https://ftpmirror.gnu.org/binutils/binutils-${BINUTILS_VERSION}.tar.xz"
IASL_ARCHIVE="https://github.com/acpica/acpica/archive/refs/tags/${IASL_VERSION}.tar.gz"
# CLANG toolchain archive locations
LLVM_ARCHIVE="https://github.com/llvm/llvm-project/releases/download/llvmorg-${CLANG_VERSION}/llvm-${CLANG_VERSION}.src.tar.xz"
CLANG_ARCHIVE="https://github.com/llvm/llvm-project/releases/download/llvmorg-${CLANG_VERSION}/clang-${CLANG_VERSION}.src.tar.xz"
CRT_ARCHIVE="https://github.com/llvm/llvm-project/releases/download/llvmorg-${CLANG_VERSION}/compiler-rt-${CLANG_VERSION}.src.tar.xz"
CTE_ARCHIVE="https://github.com/llvm/llvm-project/releases/download/llvmorg-${CLANG_VERSION}/clang-tools-extra-${CLANG_VERSION}.src.tar.xz"
LLVMCMAKE_ARCHIVE="https://github.com/llvm/llvm-project/releases/download/llvmorg-${CLANG_VERSION}/cmake-${CLANG_VERSION}.src.tar.xz"
CMAKE_ARCHIVE="https://cmake.org/files/v${CMAKE_VERSION%.*}/cmake-${CMAKE_VERSION}.tar.gz"
NASM_ARCHIVE="https://www.nasm.us/pub/nasm/releasebuilds/${NASM_VERSION}/nasm-${NASM_VERSION}.tar.bz2"
ALL_ARCHIVES="$GMP_ARCHIVE $MPFR_ARCHIVE $MPC_ARCHIVE \ # GCC toolchain archive locations
$GCC_ARCHIVE $BINUTILS_ARCHIVE $IASL_ARCHIVE \ GMP_BASE_URL="https://ftpmirror.gnu.org/gmp"
$LLVM_ARCHIVE $CLANG_ARCHIVE $LLVMCMAKE_ARCHIVE \ MPFR_BASE_URL="https://ftpmirror.gnu.org/mpfr"
$CRT_ARCHIVE $CTE_ARCHIVE $CMAKE_ARCHIVE $NASM_ARCHIVE" MPC_BASE_URL="https://ftpmirror.gnu.org/mpc"
GCC_BASE_URL="https://ftpmirror.gnu.org/gcc/gcc-${GCC_VERSION}"
BINUTILS_BASE_URL="https://ftpmirror.gnu.org/binutils"
IASL_BASE_URL="https://github.com/acpica/acpica/archive/refs/tags/"
# CLANG toolchain archive locations
LLVM_BASE_URL="https://github.com/llvm/llvm-project/releases/download/llvmorg-${CLANG_VERSION}"
CLANG_BASE_URL="https://github.com/llvm/llvm-project/releases/download/llvmorg-${CLANG_VERSION}"
CRT_BASE_URL="https://github.com/llvm/llvm-project/releases/download/llvmorg-${CLANG_VERSION}"
CTE_BASE_URL="https://github.com/llvm/llvm-project/releases/download/llvmorg-${CLANG_VERSION}"
LLVMCMAKE_BASE_URL="https://github.com/llvm/llvm-project/releases/download/llvmorg-${CLANG_VERSION}"
CMAKE_BASE_URL="https://cmake.org/files/v${CMAKE_VERSION%.*}"
NASM_BASE_URL="https://www.nasm.us/pub/nasm/releasebuilds/${NASM_VERSION}"
ALL_ARCHIVES="$GMP_BASE_URL/$GMP_ARCHIVE $MPFR_BASE_URL/$MPFR_ARCHIVE $MPC_BASE_URL/$MPC_ARCHIVE \
$GCC_BASE_URL/$GCC_ARCHIVE $BINUTILS_BASE_URL/$BINUTILS_ARCHIVE $IASL_BASE_URL/$IASL_ARCHIVE \
$LLVM_BASE_URL/$LLVM_ARCHIVE $CLANG_BASE_URL/$CLANG_ARCHIVE $LLVMCMAKE_BASE_URL/$LLVMCMAKE_ARCHIVE \
$CRT_BASE_URL/$CRT_ARCHIVE $CTE_BASE_URL/$CTE_ARCHIVE $CMAKE_BASE_URL/$CMAKE_ARCHIVE $NASM_BASE_URL/$NASM_ARCHIVE"
# GCC toolchain directories # GCC toolchain directories
GMP_DIR="gmp-${GMP_VERSION}" GMP_DIR="gmp-${GMP_VERSION}"
@ -300,23 +319,27 @@ ada_requested() {
download() { download() {
package=$1 package=$1
# shellcheck disable=SC2086 # shellcheck disable=SC2086
archive="$(eval echo \$$package"_ARCHIVE")" if [ "${USE_COREBOOT_MIRROR}" -eq 0 ]; then
url="$(eval echo \$$package"_BASE_URL")"
else
url="${COREBOOT_MIRROR_URL}"
fi
FILE=$(basename "$archive") file="$(eval echo \$$package"_ARCHIVE")"
printf " * $FILE " printf " * ${file} "
if test -f "tarballs/$FILE"; then if test -f "tarballs/${file}"; then
printf "(cached)... " printf "(cached)... "
else else
printf "(downloading from $archive)" printf "(downloading from ${url}/${file})"
rm -f "tarballs/$FILE" rm -f "tarballs/${file}"
cd tarballs || exit 1 cd tarballs || exit 1
download_showing_percentage "$archive" download_showing_percentage "${url}/${file}"
cd .. cd ..
fi fi
if [ ! -f "tarballs/$FILE" ]; then if [ ! -f "tarballs/${file}" ]; then
printf "${RED}Failed to download $FILE.${NC}\n" printf "${RED}Failed to download ${file}.${NC}\n"
exit 1 exit 1
fi fi
} }
@ -326,8 +349,7 @@ download() {
compute_hash() { compute_hash() {
package=$1 package=$1
# shellcheck disable=SC2086 # shellcheck disable=SC2086
archive="$(eval echo \$$package"_ARCHIVE")" file="$(eval echo \$$package"_ARCHIVE")"
file="$(basename "$archive")"
if test -z "$CHECKSUM"; then if test -z "$CHECKSUM"; then
echo "${RED}\$CHECKSUM program missing. This is bad.${NC}" 1>&2 echo "${RED}\$CHECKSUM program missing. This is bad.${NC}" 1>&2
@ -340,8 +362,7 @@ compute_hash() {
error_hash_missing() { error_hash_missing() {
package="$1" package="$1"
# shellcheck disable=SC2086 # shellcheck disable=SC2086
archive="$(eval echo \$$package"_ARCHIVE")" file="$(eval echo \$$package"_ARCHIVE")"
file="$(basename "$archive")"
fullhashfile="util/crossgcc/sum/$file.cksum" fullhashfile="util/crossgcc/sum/$file.cksum"
printf "${RED}hash file missing:${NC}\n\n" 1>&2 printf "${RED}hash file missing:${NC}\n\n" 1>&2
@ -357,8 +378,7 @@ error_hash_missing() {
get_known_hash() { get_known_hash() {
package=$1 package=$1
# shellcheck disable=SC2086 # shellcheck disable=SC2086
archive="$(eval echo \$$package"_ARCHIVE")" file="$(eval echo \$$package"_ARCHIVE")"
file="$(basename "$archive")"
hashfile="sum/$file.cksum" hashfile="sum/$file.cksum"
if [ ! -f "$hashfile" ]; then if [ ! -f "$hashfile" ]; then
@ -377,8 +397,7 @@ error_hash_mismatch() {
known_hash="$2" known_hash="$2"
computed_hash="$3" computed_hash="$3"
# shellcheck disable=SC2086 # shellcheck disable=SC2086
archive="$(eval echo \$$package"_ARCHIVE")" file="$(eval echo \$$package"_ARCHIVE")"
file="$(basename "$archive")"
printf "${RED}hash mismatch:${NC}\n\n" printf "${RED}hash mismatch:${NC}\n\n"
printf " expected (known) hash: $known_hash\n" printf " expected (known) hash: $known_hash\n"
@ -397,8 +416,6 @@ error_hash_mismatch() {
# hash; Bail out on mismatch or missing hash file. # hash; Bail out on mismatch or missing hash file.
verify_hash() { verify_hash() {
package=$1 package=$1
# shellcheck disable=SC2086
archive="$(eval echo \$$package"_ARCHIVE")"
known_hash="$(get_known_hash "$package")" || exit "$?" known_hash="$(get_known_hash "$package")" || exit "$?"
computed_hash="$(compute_hash "$package")" || exit "$?" computed_hash="$(compute_hash "$package")" || exit "$?"
@ -418,7 +435,7 @@ unpack_and_patch() {
# shellcheck disable=SC2086 # shellcheck disable=SC2086
dir="$(eval echo \$$package"_DIR")" dir="$(eval echo \$$package"_DIR")"
test -d "${dir}" && test -f "${dir}/.unpack_success" || ( test -d "${dir}" && test -f "${dir}/.unpack_success" || (
printf " * $(basename "$archive")\n" printf " * "$archive"\n"
FLAGS=zxf FLAGS=zxf
suffix=$(echo "$archive" | sed 's,.*\.,,') suffix=$(echo "$archive" | sed 's,.*\.,,')
if [ "$suffix" = "gz" ] && [ -n "$PIGZ" ]; then FLAGS="-I pigz -xf" if [ "$suffix" = "gz" ] && [ -n "$PIGZ" ]; then FLAGS="-I pigz -xf"
@ -429,7 +446,7 @@ unpack_and_patch() {
elif [ "$suffix" = "lzma" ]; then FLAGS="--lzma -xf" elif [ "$suffix" = "lzma" ]; then FLAGS="--lzma -xf"
fi fi
# shellcheck disable=SC2086 # shellcheck disable=SC2086
$TAR $FLAGS "tarballs/$(basename "$archive")" $TAR $FLAGS "tarballs/$archive"
for patch in patches/${dir}_*.patch; do for patch in patches/${dir}_*.patch; do
test -r "$patch" || continue test -r "$patch" || continue
printf " o $(basename "$patch")\n" printf " o $(basename "$patch")\n"
@ -915,12 +932,12 @@ getopt - > /dev/null 2>/dev/null || gcc -o getopt getopt.c
getoptbrand="$(getopt -V 2>/dev/null | sed -e '1!d' -e 's,^\(......\).*,\1,')" getoptbrand="$(getopt -V 2>/dev/null | sed -e '1!d' -e 's,^\(......\).*,\1,')"
if [ "${getoptbrand}" = "getopt" ]; then if [ "${getoptbrand}" = "getopt" ]; then
# Detected GNU getopt that supports long options. # Detected GNU getopt that supports long options.
args=$(getopt -l print-version,version,help,clean,directory:,bootstrap,bootstrap-only,platform:,languages:,package:,jobs:,destdir:,savetemps,scripting,ccache,supported:,urls,nocolor -o WVhcd:bBp:l:P:j:D:tSys:un -- "$@") args=$(getopt -l print-version,version,help,clean,directory:,bootstrap,bootstrap-only,platform:,languages:,package:,jobs:,destdir:,savetemps,scripting,ccache,supported:,urls,nocolor,mirror -o WVhcd:bBp:l:P:j:D:tSys:unm -- "$@")
getopt_ret=$? getopt_ret=$?
eval set -- "$args" eval set -- "$args"
else else
# Detected non-GNU getopt # Detected non-GNU getopt
args=$(getopt WVhcd:bBp:l:P:j:D:tSys:un $*) args=$(getopt WVhcd:bBp:l:P:j:D:tSys:unm $*)
getopt_ret=$? getopt_ret=$?
# shellcheck disable=SC2086 # shellcheck disable=SC2086
set -- $args set -- $args
@ -943,6 +960,7 @@ while true ; do
-B|--bootstrap-only) shift; BOOTSTRAPONLY=1; BOOTSTRAP=1;; -B|--bootstrap-only) shift; BOOTSTRAPONLY=1; BOOTSTRAP=1;;
-p|--platform) shift; TARGETARCH="$1"; shift;; -p|--platform) shift; TARGETARCH="$1"; shift;;
-l|--languages) shift; LANGUAGES="$1"; shift;; -l|--languages) shift; LANGUAGES="$1"; shift;;
-m|--mirror) shift; USE_COREBOOT_MIRROR=1;;
-D|--destdir) shift; DESTDIR="$1"; shift;; -D|--destdir) shift; DESTDIR="$1"; shift;;
-j|--jobs) shift; THREADS="$1"; JOBS="-j $1"; shift;; -j|--jobs) shift; THREADS="$1"; JOBS="-j $1"; shift;;
-P|--package) shift; PACKAGE="$1"; shift;; -P|--package) shift; PACKAGE="$1"; shift;;