arch-nspawn: generalize mirrorlist variable replacement

The host_mirrors value uses a sed pattern that is closely coupled with
the mirror URL pattern in-use by most Arch mirrors. For some variants,
like Arch Linux ARM, values such as $repo and $arch sit in other parts
in the mirror path. This change makes the sed replacement more generic
to accomodate for variations in mirrorlist URLs while remaining backward
compatible.

.gitignore

@@ -11,6 +11,7 @@ lddd
 makechrootpkg
 makerepropkg
 mkarchroot
+offload-build
 rebuildpkgs
 zsh_completion
 find-libdeps

Makefile

@@ -1,4 +1,4 @@
-V=20191212
+V=20200407
 
 PREFIX = /usr/local
 MANDIR = $(PREFIX)/share/man
@@ -17,12 +17,12 @@ IN_PROGS = \
 	makerepropkg \
 	mkarchroot \
 	makechrootpkg \
+	offload-build \
 	rebuildpkgs \
 	sogrep
 
 BINPROGS = \
-	$(IN_PROGS) \
-	offload-build \
+	$(IN_PROGS)
 
 CONFIGFILES = \
 	makepkg-x86_64.conf \

arch-nspawn.in

@@ -58,18 +58,21 @@ shift 1
 [[ -z $working_dir ]] && die 'Please specify a working directory.'
 
 pacconf_cmd=$(command -v pacman-conf || command -v pacconf)
+pacconf_arch=$($pacconf_cmd architecture)
 
 if (( ${#cache_dirs[@]} == 0 )); then
 	mapfile -t cache_dirs < <($pacconf_cmd --config "${pac_conf:-$working_dir/etc/pacman.conf}" CacheDir)
 fi
 
 # shellcheck disable=2016
-host_mirrors=($($pacconf_cmd --repo extra Server 2> /dev/null | sed -r 's#(.*/)extra/os/.*#\1$repo/os/$arch#'))
+host_mirrors=($($pacconf_cmd --repo extra Server 2> /dev/null | sed -r 's#'"${pacconf_arch}"'#$arch#;s#extra#$repo#'))
 
 for host_mirror in "${host_mirrors[@]}"; do
 	if [[ $host_mirror == *file://* ]]; then
-		host_mirror=$(echo "$host_mirror" | sed -r 's#file://(/.*)/\$repo/os/\$arch#\1#g')
-		in_array "$host_mirror" "${cache_dirs[@]}" || cache_dirs+=("$host_mirror")
+		host_mirror=$(echo "$host_mirror" | sed -r 's#file://(/.*)/\$repo[/.]*#\1#g')
+		for m in "$host_mirror"/pool/*/; do
+			in_array "$m" "${cache_dirs[@]}" || cache_dirs+=("$m")
+		done
 	fi
 done
 

checkpkg.in

@@ -95,8 +95,10 @@ for _pkgname in "${pkgname[@]}"; do
 
 	if (( $# )); then
 		case $1 in
+			*://*)
+				pkgurl=$1 ;;
 			/*|*/*)
-				pkgurl=file://$(readlink -m "$1") ;;
+				pkgurl=$(readlink -m "$1") ;;
 			*.pkg.tar*)
 				pkgurl=$1 ;;
 			'')

doc/makerepropkg.1.asciidoc

@@ -7,12 +7,12 @@ makerepropkg - Rebuild a package to see if it is reproducible
 
 Synopsis
 --------
-makerepropkg [OPTIONS] <package_file>
+makerepropkg [OPTIONS] <package_file>...
 
 Description
 -----------
 
-Given the path to a built pacman package, attempt to rebuild it using the
+Given the path to a built pacman package(s), attempt to rebuild it using the
 PKGBUILD in the current directory. The package will be built in an environment
 as closely matching the environment of the initial package as possible, by
 building up a chroot to match the information exposed in the package's
@@ -20,12 +20,19 @@ linkman:BUILDINFO[5] manifest. On success, the resulting package will be
 compared to the input package, and makerepropkg will report whether the
 artifacts are identical.
 
+When given multiple packages, additional package files are assumed to be split
+packages and will be treated as additional artifacts to compare during the
+verification step.
+
 This implements a verifier for pacman/libalpm packages in accordance with the
 link:https://reproducible-builds.org/[Reproducible Builds] project.
 
 Options
 -------
 
+*-d*::
+	If packages are not reproducible, compare them using diffoscope.
+
 *-c*::
 	Set the pacman cache directory.
 

makechrootpkg.in

@@ -366,11 +366,7 @@ if arch-nspawn "$copydir" \
 	"${bindmounts_ro[@]}" "${bindmounts_rw[@]}" \
 	/chrootbuild "${makepkg_args[@]}"
 then
-	pkgnames=()
-	for pkgfile in "$copydir"/pkgdest/*; do
-		pkgfile=${pkgfile##*/};
-		pkgnames+=("${pkgfile%-*-*-*}");
-	done
+	mapfile -t pkgnames < <(sudo -u "$makepkg_user" bash -c 'source PKGBUILD; printf "%s\n" "${pkgname[@]}"')
 	move_products
 else
 	(( ret += 1 ))
@@ -388,29 +384,29 @@ else
 	if (( run_checkpkg )); then
 		msg "Running checkpkg"
 
-		# sync off-site databases for up-to-date queries
-		trap 'rm -rf $dbpath; cleanup' EXIT INT TERM QUIT
-		dbpath=$(mktemp -d --tmpdir makechrootpkg-database.XXXXXXXXXX)
-		mkdir -p "$dbpath"
-		pacman -Sy --dbpath "$dbpath" --logfile /dev/null
+		mapfile -t remotepkgs < <(pacman --config "$copydir"/etc/pacman.conf \
+			--dbpath "$copydir"/var/lib/pacman \
+			-Sddp "${pkgnames[@]}")
 
-		# query current package locations
-		remotepkgs=($(pacman -Sddp --dbpath "$dbpath" --logfile /dev/null "${pkgnames[@]}"))
-		if (( $? )); then
+		if ! wait $!; then
 			warning "Skipped checkpkg due to missing repo packages"
 			exit 0
 		fi
 
 		# download package files if any non-local location exists
 		for remotepkg in "${remotepkgs[@]}"; do
-			[[ $remotepkg == file://* ]] && continue
-			msg2 "Downloading current versions"
-			pacman --noconfirm -Swdd --dbpath "$dbpath" --logfile /dev/null "${pkgnames[@]}"
-			break
+			if [[ $remotepkg != file://* ]]; then
+				msg2 "Downloading current versions"
+				arch-nspawn "$copydir" pacman --noconfirm -Swdd "${pkgnames[@]}"
+				mapfile -t remotepkgs < <(pacman --config "$copydir"/etc/pacman.conf \
+					--dbpath "$copydir"/var/lib/pacman \
+					-Sddp "${pkgnames[@]}")
+				break
+			fi
 		done
 
 		msg2 "Checking packages"
-		sudo -u "$makepkg_user" checkpkg --rmdir --warn
+		sudo -u "$makepkg_user" checkpkg --rmdir --warn "${remotepkgs[@]/#file:\/\//}"
 	fi
 	true
 fi

makepkg-x86_64.conf

@@ -132,7 +132,7 @@ DBGSRCDIR="/usr/src/debug"
 COMPRESSGZ=(gzip -c -f -n)
 COMPRESSBZ2=(bzip2 -c -f)
 COMPRESSXZ=(xz -c -z -)
-COMPRESSZST=(zstd -c -z -q -)
+COMPRESSZST=(zstd -c -T0 --ultra -20 -)
 COMPRESSLRZ=(lrzip -q)
 COMPRESSLZO=(lzop -q)
 COMPRESSZ=(compress -c -f)
@@ -143,7 +143,7 @@ COMPRESSLZ=(lzip -c -f)
 # EXTENSION DEFAULTS
 #########################################################################
 #
-PKGEXT='.pkg.tar.xz'
+PKGEXT='.pkg.tar.zst'
 SRCEXT='.src.tar.gz'
 
 # vim: set ft=sh ts=2 sw=2 et:

makerepropkg.in

@@ -29,6 +29,7 @@ declare -a buildenv buildopts installed installpkgs
 archiveurl='https://archive.archlinux.org/packages'
 buildroot=/var/lib/archbuild/reproducible
 chroot=testenv
+diffoscope=0
 
 parse_buildinfo() {
     local line var val
@@ -59,7 +60,7 @@ get_pkgfile() {
     local pkgname=${pkgfilebase%-*-*-*}
     local pkgfile ext
 
-    for ext in .xz .zstd ''; do
+    for ext in .zst .xz ''; do
         pkgfile=${pkgfilebase}.pkg.tar${ext}
 
         for c in "${cache_dirs[@]}"; do
@@ -94,19 +95,21 @@ package, including the .PKGINFO as well as the buildinfo.
 For more details see https://reproducible-builds.org/
 
 OPTIONS
+    -d            Run diffoscope if the package is unreproducible
     -c <dir>      Set pacman cache
     -M <file>     Location of a makepkg config file
     -h            Show this usage message
 __EOF__
 }
 
-while getopts 'M:c:h' arg; do
-        case "$arg" in
-                M) archroot_args+=(-M "$OPTARG") ;;
-                c) cache_dirs+=("$OPTARG") ;;
-                h) usage; exit 0 ;;
-                *|?) usage; exit 1 ;;
-        esac
+while getopts 'dM:c:h' arg; do
+    case "$arg" in
+        d) diffoscope=1 ;;
+        M) archroot_args+=(-M "$OPTARG") ;;
+        c) cache_dirs+=("$OPTARG") ;;
+        h) usage; exit 0 ;;
+        *|?) usage; exit 1 ;;
+    esac
 done
 shift $((OPTIND - 1))
 
@@ -114,10 +117,13 @@ check_root
 
 if [[ -n $1 ]]; then
     pkgfile="$1"
-    if ! bsdtar -tqf "${pkgfile}" .BUILDINFO >/dev/null 2>&1; then
-        error "file is not a valid pacman package: '%s'" "${pkgfile}"
-        exit 1
-    fi
+    splitpkgs=("$@")
+    for f in "${splitpkgs[@]}"; do
+        if ! bsdtar -tqf "${f}" .BUILDINFO >/dev/null 2>&1; then
+            error "file is not a valid pacman package: '%s'" "${f}"
+            exit 1
+        fi
+    done
 else
     error "no package file specified. Try '${BASH_SOURCE[0]##*/} -h' for more information. "
     exit 1
@@ -137,6 +143,7 @@ parse_buildinfo < <(bsdtar -xOqf "${pkgfile}" .BUILDINFO)
 export SOURCE_DATE_EPOCH="${buildinfo[builddate]}"
 PACKAGER="${buildinfo[packager]}"
 BUILDDIR="${buildinfo[builddir]}"
+PKGEXT=${pkgfile#${pkgfile%.pkg.tar*}}
 
 # nuke and restore reproducible testenv
 for copy in "${buildroot}"/*/; do
@@ -152,15 +159,14 @@ for fname in "${installed[@]}"; do
         exit 1
     fi
 done
-printf '%s\n' "${allpkgfiles[@]}" | mkarchroot -U "${archroot_args[@]}" "${buildroot}"/root - || exit 1
-
+printf '%s\n' "${allpkgfiles[@]}" | mkarchroot -M @pkgdatadir@/makepkg-x86_64.conf -U "${archroot_args[@]}" "${buildroot}"/root - || exit 1
 
 # use makechrootpkg to prep the build directory
 makechrootpkg -r "${buildroot}" -l "${chroot}" -- --packagelist || exit 1
 
 # set detected makepkg.conf options
 {
-    for var in PACKAGER BUILDDIR; do
+    for var in PACKAGER BUILDDIR PKGEXT; do
         printf '%s=%s\n' "${var}" "${!var@Q}"
     done
     printf 'OPTIONS=(%s)\n' "${buildopts[*]@Q}"
@@ -173,18 +179,26 @@ arch-nspawn "${buildroot}/${chroot}" \
     --bind="${PWD}:/startdir" \
     --bind="${SRCDEST}:/srcdest" \
     /chrootbuild -C --noconfirm --log --holdver --skipinteg
+ret=$?
 
-if (( $? == 0 )); then
+if (( ${ret} == 0 )); then
     msg2 "built succeeded! built packages can be found in ${buildroot}/${chroot}/pkgdest"
     msg "comparing artifacts..."
-    if cmp -s "${pkgfile}" "${buildroot}/${chroot}/pkgdest/${pkgfile##*/}"; then
-        msg2 "Package successfully reproduced!"
-        exit 0
-    else
-        warning "Package is not reproducible. :("
-        sha256sum "${pkgfile}" "${buildroot}/${chroot}/pkgdest/${pkgfile##*/}"
-    fi
+
+    for pkgfile in "${splitpkgs[@]}"; do
+        comparefiles=("${pkgfile}" "${buildroot}/${chroot}/pkgdest/${pkgfile##*/}")
+        if cmp -s "${comparefiles[@]}"; then
+            msg2 "Package '%s' successfully reproduced!" "${pkgfile}"
+        else
+            ret=1
+            warning "Package '%s' is not reproducible. :(" "${pkgfile}"
+            sha256sum "${comparefiles[@]}"
+            if (( diffoscope )); then
+                diffoscope "${comparefiles[@]}"
+            fi
+        fi
+    done
 fi
 
-# the package either failed to build, or was unreproducible
-exit 1
+# return failure from chrootbuild, or the reproducibility status
+exit ${ret}

offload-build.in

@@ -74,18 +74,20 @@ while (( $# )); do
 done
 
 # multilib must be handled specially
+archbuild_arch="${arch}"
 if [[ $repo = multilib* ]]; then
-    arch=
+    archbuild_arch=
 fi
 
-archbuild_cmd=("${repo}${arch:+-$arch}-build" "$@")
+archbuild_cmd=("${repo}${archbuild_arch:+-$archbuild_arch}-build" "$@")
 
-trap 'rm -rf $SRCPKGDEST' EXIT INT TERM QUIT
+trap 'rm -rf $TEMPDIR' EXIT INT TERM QUIT
 
 # Use a source-only tarball as an intermediate to transfer files. This
 # guarantees the checksums are okay, and guarantees that all needed files are
 # transferred, including local sources, install scripts, and changelogs.
-export SRCPKGDEST=$(mktemp -d)
+export TEMPDIR=$(mktemp -d --tmpdir offload-build.XXXXXXXXXX)
+export SRCPKGDEST=${TEMPDIR}
 makepkg --source || die "unable to make source package"
 
 # Temporary cosmetic workaround makepkg if SRCDEST is set somewhere else
@@ -108,14 +110,22 @@ mapfile -t files < <(
                 printf "%s\n" "" "-> build complete" &&
                 printf "\t%s\n" "$temp"/*
             } >&2 &&
-            makepkg --packagelist
+            makepkg_user_config="${XDG_CONFIG_HOME:-$HOME/.config}/pacman/makepkg.conf" &&
+            makepkg_config="/usr/share/devtools/makepkg-'"${arch}"'.conf" &&
+            if [[ -f /usr/share/devtools/makepkg-'"${repo}"'-'"${arch}"'.conf ]]; then
+                makepkg_config="/usr/share/devtools/makepkg-'"${repo}"'-'"${arch}"'.conf"
+            fi &&
+            makepkg --config <(cat "${makepkg_user_config}" "${makepkg_config}" 2>/dev/null) --packagelist &&
+            printf "%s\n" "${temp}/PKGBUILD"
 ')
 
 
 if (( ${#files[@]} )); then
     printf '%s\n' '' '-> copying files...'
     load_makepkg_config
-    scp "${files[@]/#/$server:}" "${PKGDEST:-${PWD}}/"
+    scp "${files[@]/#/$server:}" "${TEMPDIR}/"
+    mv "${TEMPDIR}"/*.pkg.tar* "${PKGDEST:-${PWD}}/"
+    mv "${TEMPDIR}/PKGBUILD" "${PWD}/"
 else
     exit 1
 fi

zsh_completion.in

@@ -94,10 +94,11 @@ _offload_build_args=(
 )
 
 _makerepropkg_args=(
+	'-d[Run diffoscope if the package is unreproducible]'
 	'-c[Set pacman cache]:pacman_cache:_files -/'
 	'-M[Location of a makepkg config file]:makepkg_config:_files -g "*.conf(.)"'
 	'-h[Display usage]'
-	'1:working_dir:_files -g "*.pkg.tar.*(.)"'
+	'*:working_dir:_files -g "*.pkg.tar.*(.)"'
 )
 
 _devtools_completions_all_packages() {