Canonicalize array initialization

* initialize cache_dirs with one of:
 1. value provided in command line -c switch
 2. pacman.conf provided in command line -C switch
 3. container pacman.conf file
* find first writable cache_dir, bind it rw, remove from cache_dirs
* extract host mirrors, find file:// ones, append to cache_dirs
* extract server url, find file:// ones, append to cache_dirs
* concatenate cache_dirs removing duplicates, bind them ro to container
* check if any extra cache_dirs was defined, append to
  container pacman.conf as new [options] section.

Fix #57 #58 #59

.gitignore

@@ -9,7 +9,9 @@ commitpkg
 finddeps
 lddd
 makechrootpkg
+makerepropkg
 mkarchroot
+offload-build
 rebuildpkgs
 zsh_completion
 find-libdeps
@@ -17,3 +19,4 @@ crossrepomove
 arch-nspawn
 sogrep
 doc/*.1
+doc/*.7

Makefile

@@ -1,27 +1,28 @@
-V=20191016
+V=20210202
 
 PREFIX = /usr/local
 MANDIR = $(PREFIX)/share/man
 
 IN_PROGS = \
-	checkpkg \
-	commitpkg \
 	archco \
+	arch-nspawn \
 	archrelease \
 	archbuild \
-	lddd \
-	finddeps \
-	rebuildpkgs \
-	find-libdeps \
+	checkpkg \
+	commitpkg \
 	crossrepomove\
-	arch-nspawn \
+	finddeps \
+	find-libdeps \
+	lddd \
+	makerepropkg \
 	mkarchroot \
 	makechrootpkg \
+	offload-build \
+	rebuildpkgs \
 	sogrep
 
 BINPROGS = \
-	$(IN_PROGS) \
-	offload-build \
+	$(IN_PROGS)
 
 CONFIGFILES = \
 	makepkg-x86_64.conf \
@@ -71,9 +72,11 @@ MANS = \
 	doc/checkpkg.1 \
 	doc/offload-build.1 \
 	doc/sogrep.1 \
+	doc/makerepropkg.1 \
 	doc/mkarchroot.1 \
 	doc/find-libdeps.1 \
-	doc/find-libprovides.1
+	doc/find-libprovides.1 \
+	doc/devtools.7
 
 
 all: $(BINPROGS) bash_completion zsh_completion man

arch-nspawn.in

@@ -20,6 +20,7 @@ umask 0022
 working_dir=''
 
 files=()
+mount_args=()
 
 usage() {
 	echo "Usage: ${0##*/} [options] working-dir [systemd-nspawn arguments]"
@@ -56,61 +57,73 @@ shift 1
 
 [[ -z $working_dir ]] && die 'Please specify a working directory.'
 
-pacconf_cmd=$(command -v pacman-conf || command -v pacconf)
-
 if (( ${#cache_dirs[@]} == 0 )); then
-	mapfile -t cache_dirs < <($pacconf_cmd --config "${pac_conf:-$working_dir/etc/pacman.conf}" CacheDir)
+	mapfile -t cache_dirs < <(pacman-conf --config "${pac_conf:-$working_dir/etc/pacman.conf}" CacheDir)
 fi
 
-# shellcheck disable=2016
-host_mirrors=($($pacconf_cmd --repo extra Server 2> /dev/null | sed -r 's#(.*/)extra/os/.*#\1$repo/os/$arch#'))
-
-while read -r line; do
-	mapfile -t lines < <($pacconf_cmd --config "${pac_conf:-$working_dir/etc/pacman.conf}" \
-		--repo $line Server | sed -r 's#(.*/)[^/]+/os/.+#\1$repo/os/$arch#')
-	if [[ ${lines[0]} != ${host_mirrors[0]} ]]; then
-		for line in "${lines[@]}"; do
-			if [[ $line = file://* ]]; then
-				line=${line#file://}
-				in_array "$line" "${cache_dirs[@]}" || cache_dirs+=("$line")
-			fi
-		done
+# find first writable cache, bind it to container in rw mode, remove it from cache_dirs
+for cache_dir in "${cache_dirs[@]}"; do
+	if [[ -d "$cache_dir" && -w "$cache_dir" ]]; then
+		mount_args+=("--bind=$cache_dir")
+		mapfile -t cache_dirs < <(printf "%s\n" "${cache_dirs[@]}" | grep -vFx "$cache_dir")
+		break
 	fi
-done < <($pacconf_cmd --config "${pac_conf:-$working_dir/etc/pacman.conf}" --repo-list)
+# this could print an warning to the user that no host caches would be used
+done
 
-# {{{ functions
-build_mount_args() {
-	declare -g mount_args=()
+# shellcheck disable=2016
+mapfile -t host_mirrors < <(pacman-conf --repo extra Server 2> /dev/null | sed -r 's#(.*/)extra/os/.*#\1$repo/os/$arch#')
 
-	for host_mirror in "${host_mirrors[@]}"; do
-		if [[ $host_mirror == *file://* ]]; then
-			host_mirror_path=$(echo "$host_mirror" | sed -r 's#file://(/.*)/\$repo/os/\$arch#\1#g')
-			mount_args+=("--bind-ro=${host_mirror_path//:/\\:}")
-		fi
+# extract local host mirrors (Server=file://...)
+mapfile -t host_mirrors_local < <(printf "%s\n" "${host_mirrors[@]}" | grep -Po "(?<=file://).*(?=\/\$repo)")
+
+for host_mirror in "${host_mirrors_local[@]}"; do
+	for m in "$host_mirror"/pool/*/; do
+		cache_dirs+=("$m")
 	done
+done
 
-	mount_args+=("--bind=${cache_dirs[0]//:/\\:}")
+mapfile -t repos < <(pacman-conf --config "${pac_conf:-$working_dir/etc/pacman.conf}" --repo-list)
 
-	for cache_dir in "${cache_dirs[@]:1}"; do
+declare -a servers
+for repo in "${repos[@]}"; do
+	mapfile -t -O "${#servers[@]}" servers < <( \
+		pacman-conf --config "${pac_conf:-$working_dir/etc/pacman.conf}" --repo "$repo" Server | \
+		grep -Po "(?<=file://).*$" | sed -r -e "s#\$repo#$repo#" -e 's#/$arch##' )
+done
+
+cache_dirs+=("${servers[@]}")
+
+# rest of cache_dirs are getting bind ro therefor we don't care about ordering and can use sort -u for concatenation
+if (( ${#cache_dirs[@]} != 0 )); then
+	mapfile -t cache_dirs < <(printf '%s\n' "${cache_dirs[@]}" | sort -u)
+	for cache_dir in "${cache_dirs[@]}"; do
 		mount_args+=("--bind-ro=${cache_dir//:/\\:}")
 	done
-}
+fi
 
+# {{{ functions
 copy_hostconf () {
 	unshare --fork --pid gpg --homedir "$working_dir"/etc/pacman.d/gnupg/ --no-permission-warning --quiet --batch --import --import-options import-local-sigs "$(pacman-conf GpgDir)"/pubring.gpg >/dev/null 2>&1
 	pacman-key --gpgdir "$working_dir"/etc/pacman.d/gnupg/ --import-trustdb "$(pacman-conf GpgDir)" >/dev/null 2>&1
+
 	printf 'Server = %s\n' "${host_mirrors[@]}" >"$working_dir/etc/pacman.d/mirrorlist"
 
 	[[ -n $pac_conf ]] && cp "$pac_conf" "$working_dir/etc/pacman.conf"
 	[[ -n $makepkg_conf ]] && cp "$makepkg_conf" "$working_dir/etc/makepkg.conf"
 
+	# check if any extra cache_dirs was added
+	mapfile -t container_cache_dirs < <(pacman-conf --config "$working_dir/etc/pacman.conf" CacheDir)
+	mapfile -t extra_cache_dirs < <(comm -13 <(printf "%s\n" "${container_cache_dirs[@]}"|sort) <(printf "%s\n" "${cache_dirs[@]}"|sort))
+	if (( ${#extra_cache_dirs[@]} != 0 )); then
+		echo -e "\n[options]\nCacheDir = ${extra_cache_dirs[*]}" >> "$working_dir/etc/pacman.conf"
+	fi
+
 	local file
 	for file in "${files[@]}"; do
 		mkdir -p "$(dirname "$working_dir$file")"
 		cp -T "$file" "$working_dir$file"
 	done
-
-	sed -r "s|^#?\\s*CacheDir.+|CacheDir = ${cache_dirs[*]}|g" -i "$working_dir/etc/pacman.conf"
 }
 # }}}
 
@@ -123,7 +136,6 @@ elif [[ $(cat "$working_dir/.arch-chroot") != "$CHROOT_VERSION" ]]; then
 	die "chroot '%s' is not at version %s. Please rebuild." "$working_dir" "$CHROOT_VERSION"
 fi
 
-build_mount_args
 copy_hostconf
 
 eval "$(grep -a '^CARCH=' "$working_dir/etc/makepkg.conf")"

archbuild.in

@@ -68,7 +68,7 @@ if ${clean_first} || [[ ! -d "${chroots}/${repo}-${arch}" ]]; then
 	lock_close 9
 
 	rm -rf --one-file-system "${chroots}/${repo}-${arch}"
-	mkdir -m755 -p "${chroots}/${repo}-${arch}"
+	(umask 0022; mkdir -p "${chroots}/${repo}-${arch}")
 	setarch "${arch}" mkarchroot \
 		-C "${pacman_config}" \
 		-M "${makepkg_config}" \

checkpkg.in

@@ -95,8 +95,10 @@ for _pkgname in "${pkgname[@]}"; do
 
 	if (( $# )); then
 		case $1 in
+			*://*)
+				pkgurl=$1 ;;
 			/*|*/*)
-				pkgurl=file://$(readlink -m "$1") ;;
+				pkgurl=$(readlink -m "$1") ;;
 			*.pkg.tar*)
 				pkgurl=$1 ;;
 			'')

commitpkg.in

@@ -109,10 +109,10 @@ if [[ -z $server ]]; then
 fi
 
 if [[ -n $(svn status -q) ]]; then
-	msgtemplate="upgpkg: $pkgbase $(get_full_version)"$'\n\n'
+	msgtemplate="upgpkg: $pkgbase $(get_full_version)"
 	if [[ -n $1 ]]; then
 		stat_busy 'Committing changes to trunk'
-		svn commit -q -m "${msgtemplate}${1}" || die
+		svn commit -q -m "${msgtemplate}: ${1}" || die
 		stat_done
 	else
 		msgfile="$(mktemp)"

doc/devtools.7.asciidoc

@@ -0,0 +1,46 @@
+devtools(7)
+===========
+
+Name
+----
+devtools - Developer tools for the Arch Linux distribution
+
+Description
+-----------
+
+Devtools contains tools for package maintenance in Arch Linux. The toolset
+varies from tools for building packages in a clean chroot ('mkarchroot',...),
+packaging related tools for sonames ('sogrep', 'lddd') and tools for
+repository management such as ('archco', 'extra2community')
+
+Programs
+--------
+The list below gives a short overview; see the respective documentation
+for details.
+
+linkman:checkpkg[1]
+	Compare the current build pakcage with the repository version
+
+linkman:find-libdeps[1]
+	Find soname dependencies for a package
+
+linkman:find-libprovides[1]
+	Find soname's which are provided by a package
+
+linkman:lddd[1]
+	Find broken library links on your system
+
+linkman:makerepropkg[1]
+	Rebuild a package to see if it is reproducible
+
+linkman:mkarchroot[1]
+	Creates an arch chroot in a specified location with a specified set of
+	packages
+
+linkman:offload-build[1]
+	Build a PKGBUILD on a remote server using makechrootpkg
+
+linkman:sogrep[1]
+	Find packages using a linked to a given shared library
+
+include::footer.asciidoc[]

doc/makerepropkg.1.asciidoc

@@ -0,0 +1,54 @@
+makerepropkg(1)
+================
+
+Name
+----
+makerepropkg - Rebuild a package to see if it is reproducible
+
+Synopsis
+--------
+makerepropkg [OPTIONS] [<package_file|pkgname>...]
+
+Description
+-----------
+
+Given the path to a built pacman package(s), attempt to rebuild it using the
+PKGBUILD in the current directory. The package will be built in an environment
+as closely matching the environment of the initial package as possible, by
+building up a chroot to match the information exposed in the package's
+linkman:BUILDINFO[5] manifest. On success, the resulting package will be
+compared to the input package, and makerepropkg will report whether the
+artifacts are identical.
+
+When given multiple packages, additional package files are assumed to be split
+packages and will be treated as additional artifacts to compare during the
+verification step.
+
+A valid target(s) for pacman -S can be specified instead, and makerepropkg will
+download it to the cache if needed. This is mostly useful to specify which
+repository to retrieve from. If no positional arguments are specified, the
+targets will be sourced from the PKGBUILD.
+
+In either case, the package name will be converted to a filename from the
+cache, and makerepropkg will proceed as though this filename was initially
+specified.
+
+This implements a verifier for pacman/libalpm packages in accordance with the
+link:https://reproducible-builds.org/[Reproducible Builds] project.
+
+Options
+-------
+
+*-d*::
+	If packages are not reproducible, compare them using diffoscope.
+
+*-c*::
+	Set the pacman cache directory.
+
+*-M* <file>::
+	Location of a makepkg config file.
+
+*-h*::
+	Show this usage message
+
+include::footer.asciidoc[]

doc/mkarchroot.1.asciidoc

@@ -20,6 +20,9 @@ en_US.UTF-8 and de_DE.UTF-8 locale and a generated machine-id.
 Options
 -------
 
+*-U*::
+	Use 'pacman -U' to install packages.
+
 *-C* <file>::
 	Location of a pacman config file.
 

doc/offload-build.1.asciidoc

@@ -28,7 +28,7 @@ Options
 	architecture officially supported by Arch Linux.
 
 *-s, --server* <hostname>::
-	Offload to a specific build server. The default is dragon.archlinux.org
+	Offload to a specific build server. The default is build.archlinux.org
 	which is used as part of the build toolchain for the official Arch Linux
 	repos.
 

lib/common.sh

@@ -17,7 +17,7 @@ export LANG=C
 shopt -s extglob
 
 # check if messages are to be printed using color
-if [[ -t 2 ]]; then
+if [[ -t 2 && "$TERM" != dumb ]]; then
 	colorize
 else
 	# shellcheck disable=2034
@@ -142,7 +142,7 @@ find_cached_package() {
 	for dir in "${searchdirs[@]}"; do
 		[[ -d $dir ]] || continue
 
-		for pkg in "$dir"/*.pkg.tar?(.?z); do
+		for pkg in "$dir"/*.pkg.tar?(.!(sig|*.*)); do
 			[[ -f $pkg ]] || continue
 
 			# avoid adding duplicates of the same inode
@@ -152,7 +152,7 @@ find_cached_package() {
 
 			# split apart package filename into parts
 			pkgbasename=${pkg##*/}
-			pkgbasename=${pkgbasename%.pkg.tar?(.?z)}
+			pkgbasename=${pkgbasename%.pkg.tar*}
 
 			arch=${pkgbasename##*-}
 			pkgbasename=${pkgbasename%-"$arch"}

makechrootpkg.in

@@ -13,6 +13,8 @@
 m4_include(lib/common.sh)
 m4_include(lib/archroot.sh)
 
+source /usr/share/makepkg/util/config.sh
+
 shopt -s nullglob
 
 default_makepkg_args=(--syncdeps --noconfirm --log --holdver --skipinteg)
@@ -80,26 +82,6 @@ usage() {
 }
 
 # {{{ functions
-# Usage: load_vars $makepkg_conf
-# Globals:
-#  - SRCDEST
-#  - SRCPKGDEST
-#  - PKGDEST
-#  - LOGDEST
-#  - MAKEFLAGS
-#  - PACKAGER
-load_vars() {
-	local makepkg_conf="$1" var
-
-	[[ -f $makepkg_conf ]] || return 1
-
-	for var in {SRC,SRCPKG,PKG,LOG}DEST MAKEFLAGS PACKAGER; do
-		[[ -z ${!var:-} ]] && eval "$(source "$makepkg_conf"; printf "%s='%s'" "$var" "${!var}")"
-	done
-
-	return 0
-}
-
 # Usage: sync_chroot $chrootdir $copydir [$copy]
 sync_chroot() {
 	local chrootdir=$1
@@ -260,6 +242,15 @@ download_sources() {
 		die "Could not download sources."
 }
 
+move_logfiles() {
+	local l
+	for l in "$copydir"/logdest/*; do
+		[[ $l == */logpipe.* ]] && continue
+		chown "$src_owner" "$l"
+		mv "$l" "$LOGDEST"
+	done
+}
+
 move_products() {
 	local pkgfile
 	for pkgfile in "$copydir"/pkgdest/*; do
@@ -272,12 +263,7 @@ move_products() {
 		fi
 	done
 
-	local l
-	for l in "$copydir"/logdest/*; do
-		[[ $l == */logpipe.* ]] && continue
-		chown "$src_owner" "$l"
-		mv "$l" "$LOGDEST"
-	done
+	move_logfiles
 
 	for s in "$copydir"/srcpkgdest/*; do
 		chown "$src_owner" "$s"
@@ -338,16 +324,12 @@ for arg in "${@:$OPTIND}"; do
 	esac
 done
 
-if [[ -n $SUDO_USER ]]; then
-	eval "USER_HOME=~$SUDO_USER"
-else
-	USER_HOME=$HOME
-fi
-
 umask 0022
 
-load_vars "${XDG_CONFIG_HOME:-$USER_HOME/.config}/pacman/makepkg.conf" || load_vars "$USER_HOME/.makepkg.conf"
-load_vars /etc/makepkg.conf
+ORIG_HOME=$HOME
+IFS=: read -r _ _ _ _ _ HOME _ < <(getent passwd "${SUDO_USER:-$USER}")
+load_makepkg_config
+HOME=$ORIG_HOME
 
 # Use PKGBUILD directory if these don't exist
 [[ -d $PKGDEST ]]    || PKGDEST=$PWD
@@ -388,14 +370,11 @@ if arch-nspawn "$copydir" \
 	"${bindmounts_ro[@]}" "${bindmounts_rw[@]}" \
 	/chrootbuild "${makepkg_args[@]}"
 then
-	pkgnames=()
-	for pkgfile in "$copydir"/pkgdest/*; do
-		pkgfile=${pkgfile##*/};
-		pkgnames+=("${pkgfile%-*-*-*}");
-	done
+	mapfile -t pkgnames < <(sudo -u "$makepkg_user" bash -c 'source PKGBUILD; printf "%s\n" "${pkgname[@]}"')
 	move_products
 else
 	(( ret += 1 ))
+	move_logfiles
 fi
 
 (( temp_chroot )) && delete_chroot "$copydir" "$copy"
@@ -409,13 +388,30 @@ if (( ret != 0 )); then
 else
 	if (( run_checkpkg )); then
 		msg "Running checkpkg"
-		msg2 "Downloading current versions"
-		if pacman --noconfirm -Swdd --logfile /dev/null "${pkgnames[@]}"; then
-			msg2 "Checking packages"
-			sudo -u "$makepkg_user" checkpkg --rmdir --warn
-		else
-			warning "Skipped checkpkg due to missing packages"
+
+		mapfile -t remotepkgs < <(pacman --config "$copydir"/etc/pacman.conf \
+			--dbpath "$copydir"/var/lib/pacman \
+			-Sddp "${pkgnames[@]}")
+
+		if ! wait $!; then
+			warning "Skipped checkpkg due to missing repo packages"
+			exit 0
 		fi
+
+		# download package files if any non-local location exists
+		for remotepkg in "${remotepkgs[@]}"; do
+			if [[ $remotepkg != file://* ]]; then
+				msg2 "Downloading current versions"
+				arch-nspawn "$copydir" pacman --noconfirm -Swdd "${pkgnames[@]}"
+				mapfile -t remotepkgs < <(pacman --config "$copydir"/etc/pacman.conf \
+					--dbpath "$copydir"/var/lib/pacman \
+					-Sddp "${pkgnames[@]}")
+				break
+			fi
+		done
+
+		msg2 "Checking packages"
+		sudo -u "$makepkg_user" checkpkg --rmdir --warn "${remotepkgs[@]/#file:\/\//}"
 	fi
 	true
 fi

makepkg-x86_64.conf

@@ -15,7 +15,7 @@ DLAGENTS=('file::/usr/bin/curl -gqC - -o %o %u'
           'ftp::/usr/bin/curl -gqfC - --ftp-pasv --retry 3 --retry-delay 3 -o %o %u'
           'http::/usr/bin/curl -gqb "" -fLC - --retry 3 --retry-delay 3 -o %o %u'
           'https::/usr/bin/curl -gqb "" -fLC - --retry 3 --retry-delay 3 -o %o %u'
-          'rsync::/usr/bin/rsync --no-motd -z %u %o'
+          'rsync::/usr/bin/rsync --no-motd -zz %u %o'
           'scp::/usr/bin/scp -C %u %o')
 
 # Other common tools:
@@ -132,18 +132,18 @@ DBGSRCDIR="/usr/src/debug"
 COMPRESSGZ=(gzip -c -f -n)
 COMPRESSBZ2=(bzip2 -c -f)
 COMPRESSXZ=(xz -c -z -)
+COMPRESSZST=(zstd -c -T0 --ultra -20 -)
 COMPRESSLRZ=(lrzip -q)
 COMPRESSLZO=(lzop -q)
 COMPRESSZ=(compress -c -f)
+COMPRESSLZ4=(lz4 -q)
+COMPRESSLZ=(lzip -c -f)
 
 #########################################################################
 # EXTENSION DEFAULTS
 #########################################################################
 #
-# WARNING: Do NOT modify these variables unless you know what you are
-#          doing.
-#
-PKGEXT='.pkg.tar.xz'
+PKGEXT='.pkg.tar.zst'
 SRCEXT='.src.tar.gz'
 
 # vim: set ft=sh ts=2 sw=2 et:

makerepropkg.in

@@ -0,0 +1,239 @@
+#!/bin/bash
+#   makerepropkg - rebuild a package to see if it is reproducible
+#
+#   Copyright (c) 2019 by Eli Schwartz <eschwartz@archlinux.org>
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; either version 2 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+
+m4_include(lib/common.sh)
+m4_include(lib/archroot.sh)
+
+source /usr/share/makepkg/util/config.sh
+source /usr/share/makepkg/util/message.sh
+
+declare -A buildinfo
+declare -a buildenv buildopts installed installpkgs
+
+archiveurl='https://archive.archlinux.org/packages'
+buildroot=/var/lib/archbuild/reproducible
+chroot=testenv
+diffoscope=0
+
+parse_buildinfo() {
+    local line var val
+
+    while read -r line; do
+        var="${line%% = *}"
+        val="${line#* = }"
+        case ${var} in
+            buildenv)
+                buildenv+=("${val}")
+                ;;
+            options)
+                buildopts+=("${val}")
+                ;;
+            installed)
+                installed+=("${val}")
+                ;;
+            *)
+                buildinfo["${var}"]="${val}"
+                ;;
+        esac
+    done
+}
+
+get_pkgfile() {
+    local cdir=${cache_dirs[0]}
+    local pkgfilebase=${1}
+    local mode=${2}
+    local pkgname=${pkgfilebase%-*-*-*}
+    local pkgfile ext
+
+    # try without downloading
+    if [[ ${mode} != localonly ]] && get_pkgfile "${pkgfilebase}" localonly; then
+        return 0
+    fi
+
+    for ext in .zst .xz ''; do
+        pkgfile=${pkgfilebase}.pkg.tar${ext}
+
+        for c in "${cache_dirs[@]}"; do
+            if [[ -f ${c}/${pkgfile} ]]; then
+                cdir=${c}
+                break
+            fi
+        done
+
+        for f in "${pkgfile}" "${pkgfile}.sig"; do
+            if [[ ! -f "${cdir}/${f}" ]]; then
+                if [[ ${mode} = localonly ]]; then
+                    continue 2
+                fi
+                msg2 "retrieving '%s'..." "${f}" >&2
+                curl -Llf -# -o "${cdir}/${f}" "${archiveurl}/${pkgname:0:1}/${pkgname}/${f}" || continue 2
+            fi
+        done
+        printf '%s\n' "file://${cdir}/${pkgfile}"
+        return 0
+    done
+
+    return 1
+}
+
+usage() {
+    cat << __EOF__
+usage: ${BASH_SOURCE[0]##*/} [options] <package_file>
+
+Run this script in a PKGBUILD dir to build a package inside a
+clean chroot while attempting to reproduce it. The package file
+will be used to derive metadata needed for reproducing the
+package, including the .PKGINFO as well as the buildinfo.
+
+For more details see https://reproducible-builds.org/
+
+OPTIONS
+    -d            Run diffoscope if the package is unreproducible
+    -c <dir>      Set pacman cache
+    -M <file>     Location of a makepkg config file
+    -h            Show this usage message
+__EOF__
+}
+
+while getopts 'dM:c:h' arg; do
+    case "$arg" in
+        d) diffoscope=1 ;;
+        M) archroot_args+=(-M "$OPTARG") ;;
+        c) cache_dirs+=("$OPTARG") ;;
+        h) usage; exit 0 ;;
+        *|?) usage; exit 1 ;;
+    esac
+done
+shift $((OPTIND - 1))
+
+check_root
+
+[[ -f PKGBUILD ]] || { error "No PKGBUILD in current directory."; exit 1; }
+
+# without arguments, get list of packages from PKGBUILD
+if [[ -z $1 ]]; then
+    mapfile -t pkgnames < <(source PKGBUILD; pacman -Sddp --print-format '%r/%n' "${pkgname[@]}")
+    wait $! || {
+        error "No package file specified and failed to retrieve package names from './PKGBUILD'."
+        plain "Try '${BASH_SOURCE[0]##*/} -h' for more information." >&2
+        exit 1
+    }
+    msg "Reproducing all pkgnames listed in ./PKGBUILD"
+    set -- "${pkgnames[@]}"
+fi
+
+# check each package to see if it's a file, and if not, try to download it
+# using pacman -Sw, and get the filename from there
+splitpkgs=()
+for p in "$@"; do
+    if [[ -f ${p} ]]; then
+        splitpkgs+=("${p}")
+    else
+        pkgfile_remote=$(pacman -Sddp "${p}" 2>/dev/null) || { error "package name '%s' not in repos" "${p}"; exit 1; }
+        pkgfile=${pkgfile_remote#file://}
+        if [[ ! -f ${pkgfile} ]]; then
+            msg "Downloading package '%s' into pacman's cache" "${pkgfile}"
+            sudo pacman -Swdd --noconfirm --logfile /dev/null "${p}" || exit 1
+            pkgfile_remote=$(pacman -Sddp "${p}" 2>/dev/null)
+            pkgfile="${pkgfile_remote#file://}"
+        fi
+        splitpkgs+=("${pkgfile}")
+    fi
+done
+
+for f in "${splitpkgs[@]}"; do
+    if ! bsdtar -tqf "${f}" .BUILDINFO >/dev/null 2>&1; then
+        error "file is not a valid pacman package: '%s'" "${f}"
+        exit 1
+    fi
+done
+
+if (( ${#cache_dirs[@]} == 0 )); then
+	mapfile -t cache_dirs < <(pacman-conf CacheDir)
+fi
+
+ORIG_HOME=${HOME}
+IFS=: read -r _ _ _ _ _ HOME _ < <(getent passwd "${SUDO_USER:-$USER}")
+load_makepkg_config
+HOME=${ORIG_HOME}
+[[ -d ${SRCDEST} ]] || SRCDEST=${PWD}
+
+parse_buildinfo < <(bsdtar -xOqf "${splitpkgs[0]}" .BUILDINFO)
+export SOURCE_DATE_EPOCH="${buildinfo[builddate]}"
+PACKAGER="${buildinfo[packager]}"
+BUILDDIR="${buildinfo[builddir]}"
+PKGEXT=${splitpkgs[0]#${splitpkgs[0]%.pkg.tar*}}
+
+# nuke and restore reproducible testenv
+for copy in "${buildroot}"/*/; do
+    [[ -d ${copy} ]] || continue
+    subvolume_delete_recursive "${copy}"
+done
+rm -rf --one-file-system "${buildroot}"
+(umask 0022; mkdir -p "${buildroot}")
+
+for fname in "${installed[@]}"; do
+    if ! allpkgfiles+=("$(get_pkgfile "${fname}")"); then
+        error "failed to retrieve ${fname}"
+        exit 1
+    fi
+done
+printf '%s\n' "${allpkgfiles[@]}" | mkarchroot -M @pkgdatadir@/makepkg-x86_64.conf -U "${archroot_args[@]}" "${buildroot}"/root - || exit 1
+
+# use makechrootpkg to prep the build directory
+makechrootpkg -r "${buildroot}" -l "${chroot}" -- --packagelist || exit 1
+
+# set detected makepkg.conf options
+{
+    for var in PACKAGER BUILDDIR PKGEXT; do
+        printf '%s=%s\n' "${var}" "${!var@Q}"
+    done
+    printf 'OPTIONS=(%s)\n' "${buildopts[*]@Q}"
+    printf 'BUILDENV=(%s)\n' "${buildenv[*]@Q}"
+} >> "${buildroot}/${chroot}"/etc/makepkg.conf >> "${buildroot}/${chroot}"/etc/makepkg.conf
+install -d -o "${SUDO_UID:-$UID}" -g "$(id -g "${SUDO_UID:-$UID}")" "${buildroot}/${chroot}/${BUILDDIR}"
+
+# kick off the build
+arch-nspawn "${buildroot}/${chroot}" \
+    --bind="${PWD}:/startdir" \
+    --bind="${SRCDEST}:/srcdest" \
+    /chrootbuild -C --noconfirm --log --holdver --skipinteg
+ret=$?
+
+if (( ${ret} == 0 )); then
+    msg2 "built succeeded! built packages can be found in ${buildroot}/${chroot}/pkgdest"
+    msg "comparing artifacts..."
+
+    for pkgfile in "${splitpkgs[@]}"; do
+        comparefiles=("${pkgfile}" "${buildroot}/${chroot}/pkgdest/${pkgfile##*/}")
+        if cmp -s "${comparefiles[@]}"; then
+            msg2 "Package '%s' successfully reproduced!" "${pkgfile}"
+        else
+            ret=1
+            warning "Package '%s' is not reproducible. :(" "${pkgfile}"
+            sha256sum "${comparefiles[@]}"
+            if (( diffoscope )); then
+                diffoscope "${comparefiles[@]}"
+            fi
+        fi
+    done
+fi
+
+# return failure from chrootbuild, or the reproducibility status
+exit ${ret}

mkarchroot.in

@@ -18,6 +18,7 @@ m4_include(lib/archroot.sh)
 umask 0022
 
 working_dir=''
+umode=''
 
 files=()
 nspawn_args=()
@@ -25,6 +26,7 @@ nspawn_args=()
 usage() {
 	echo "Usage: ${0##*/} [options] working-dir package-list..."
 	echo ' options:'
+	echo '    -U            Use pacman -U to install packages'
 	echo '    -C <file>     Location of a pacman config file'
 	echo '    -M <file>     Location of a makepkg config file'
 	echo '    -c <dir>      Set pacman cache'
@@ -34,8 +36,9 @@ usage() {
 	exit 1
 }
 
-while getopts 'hC:M:c:f:s' arg; do
+while getopts 'hUC:M:c:f:s' arg; do
 	case "$arg" in
+		U) umode=U ;;
 		C) pac_conf="$OPTARG" ;;
 		M) makepkg_conf="$OPTARG" ;;
 		c) cache_dirs+=("$OPTARG") ;;
@@ -44,8 +47,10 @@ while getopts 'hC:M:c:f:s' arg; do
 		h|?) usage ;;
 		*) error "invalid argument '%s'" "$arg"; usage ;;
 	esac
-    nspawn_args+=("-$arg")
-    [[ -v OPTARG ]] && nspawn_args+=("$OPTARG")
+	if [[ $arg != U ]]; then
+		nspawn_args+=("-$arg")
+		[[ -v OPTARG ]] && nspawn_args+=("$OPTARG")
+	fi
 done
 shift $((OPTIND - 1))
 
@@ -85,7 +90,7 @@ for file in "${files[@]}"; do
 	cp "$file" "$working_dir$file"
 done
 
-pacstrap -Mcd ${pac_conf:+-C "$pac_conf"} "$working_dir" \
+pacstrap -${umode}Mcd ${pac_conf:+-C "$pac_conf"} "$working_dir" \
 	"${cache_dirs[@]/#/--cachedir=}" "$@" || die 'Failed to install all packages'
 
 printf '%s.UTF-8 UTF-8\n' en_US de_DE > "$working_dir/etc/locale.gen"

offload-build.in

@@ -18,11 +18,13 @@
 #   along with this program.  If not, see <https://www.gnu.org/licenses/>.
 #
 
+source /usr/share/makepkg/util/config.sh
+
 
 # global defaults suitable for use by Arch staff
 repo=extra
 arch=x86_64
-server=dragon.archlinux.org
+server=build.archlinux.org
 
 die() { printf "error: $1\n" "${@:2}"; exit 1; }
 
@@ -72,25 +74,34 @@ while (( $# )); do
 done
 
 # multilib must be handled specially
+archbuild_arch="${arch}"
 if [[ $repo = multilib* ]]; then
-    arch=
+    archbuild_arch=
 fi
 
-archbuild_cmd=("${repo}${arch:+-$arch}-build" "$@")
+archbuild_cmd=("${repo}${archbuild_arch:+-$archbuild_arch}-build" "$@")
 
-trap 'rm -rf $SRCPKGDEST' EXIT
+trap 'rm -rf $TEMPDIR' EXIT INT TERM QUIT
+
+# Load makepkg.conf variables to be available
+load_makepkg_config
 
 # Use a source-only tarball as an intermediate to transfer files. This
 # guarantees the checksums are okay, and guarantees that all needed files are
 # transferred, including local sources, install scripts, and changelogs.
-export SRCPKGDEST=$(mktemp -d)
+export TEMPDIR=$(mktemp -d --tmpdir offload-build.XXXXXXXXXX)
+export SRCPKGDEST=${TEMPDIR}
 makepkg --source || die "unable to make source package"
 
+# Temporary cosmetic workaround makepkg if SRCDEST is set somewhere else
+# but an empty src dir is created in PWD. Remove once fixed in makepkg.
+rmdir --ignore-fail-on-non-empty src 2>/dev/null || true
+
 mapfile -t files < <(
     # This is sort of bash golfing but it allows running a mildly complex
     # command over ssh with a single connection.
     # shellcheck disable=SC2145
-    cat "$SRCPKGDEST"/*.src.tar.gz |
+    cat "$SRCPKGDEST"/*"$SRCEXT" |
         ssh $server '
             temp="${XDG_CACHE_HOME:-$HOME/.cache}/offload-build" &&
             mkdir -p "$temp" &&
@@ -102,7 +113,21 @@ mapfile -t files < <(
                 printf "%s\n" "" "-> build complete" &&
                 printf "\t%s\n" "$temp"/*
             } >&2 &&
-            makepkg --packagelist
+            makepkg_user_config="${XDG_CONFIG_HOME:-$HOME/.config}/pacman/makepkg.conf" &&
+            makepkg_config="/usr/share/devtools/makepkg-'"${arch}"'.conf" &&
+            if [[ -f /usr/share/devtools/makepkg-'"${repo}"'-'"${arch}"'.conf ]]; then
+                makepkg_config="/usr/share/devtools/makepkg-'"${repo}"'-'"${arch}"'.conf"
+            fi &&
+            makepkg --config <(cat "${makepkg_user_config}" "${makepkg_config}" 2>/dev/null) --packagelist &&
+            printf "%s\n" "${temp}/PKGBUILD"
 ')
 
-(( ${#files[@]} )) && printf '%s\n' '' '-> copying files...' && scp "${files[@]/#/$server:}" .
+
+if (( ${#files[@]} )); then
+    printf '%s\n' '' '-> copying files...'
+    scp "${files[@]/#/$server:}" "${TEMPDIR}/"
+    mv "${TEMPDIR}"/*.pkg.tar* "${PKGDEST:-${PWD}}/"
+    mv "${TEMPDIR}/PKGBUILD" "${PWD}/"
+else
+    exit 1
+fi

pacman-extra.conf

@@ -16,10 +16,9 @@
 #GPGDir      = /etc/pacman.d/gnupg/
 #HookDir     = /etc/pacman.d/hooks/
 HoldPkg     = pacman glibc
-#XferCommand = /usr/bin/curl -C - -f %u > %o
+#XferCommand = /usr/bin/curl -L -C - -f -o %o %u
 #XferCommand = /usr/bin/wget --passive-ftp -c -O %o %u
 #CleanMethod = KeepInstalled
-#UseDelta    = 0.7
 Architecture = auto
 
 # Pacman won't upgrade packages listed in IgnorePkg and members of IgnoreGroup
@@ -90,4 +89,3 @@ Include = /etc/pacman.d/mirrorlist
 #[custom]
 #SigLevel = Optional TrustAll
 #Server = file:///home/custompkgs
-

pacman-gnome-unstable.conf

@@ -16,10 +16,9 @@
 #GPGDir      = /etc/pacman.d/gnupg/
 #HookDir     = /etc/pacman.d/hooks/
 HoldPkg     = pacman glibc
-#XferCommand = /usr/bin/curl -C - -f %u > %o
+#XferCommand = /usr/bin/curl -L -C - -f -o %o %u
 #XferCommand = /usr/bin/wget --passive-ftp -c -O %o %u
 #CleanMethod = KeepInstalled
-#UseDelta    = 0.7
 Architecture = auto
 
 # Pacman won't upgrade packages listed in IgnorePkg and members of IgnoreGroup
@@ -99,4 +98,3 @@ Include = /etc/pacman.d/mirrorlist
 #[custom]
 #SigLevel = Optional TrustAll
 #Server = file:///home/custompkgs
-

pacman-kde-unstable.conf

@@ -16,10 +16,9 @@
 #GPGDir      = /etc/pacman.d/gnupg/
 #HookDir     = /etc/pacman.d/hooks/
 HoldPkg     = pacman glibc
-#XferCommand = /usr/bin/curl -C - -f %u > %o
+#XferCommand = /usr/bin/curl -L -C - -f -o %o %u
 #XferCommand = /usr/bin/wget --passive-ftp -c -O %o %u
 #CleanMethod = KeepInstalled
-#UseDelta    = 0.7
 Architecture = auto
 
 # Pacman won't upgrade packages listed in IgnorePkg and members of IgnoreGroup
@@ -93,4 +92,3 @@ Include = /etc/pacman.d/mirrorlist
 #[custom]
 #SigLevel = Optional TrustAll
 #Server = file:///home/custompkgs
-

pacman-multilib-staging.conf

@@ -16,10 +16,9 @@
 #GPGDir      = /etc/pacman.d/gnupg/
 #HookDir     = /etc/pacman.d/hooks/
 HoldPkg     = pacman glibc
-#XferCommand = /usr/bin/curl -C - -f %u > %o
+#XferCommand = /usr/bin/curl -L -C - -f -o %o %u
 #XferCommand = /usr/bin/wget --passive-ftp -c -O %o %u
 #CleanMethod = KeepInstalled
-#UseDelta    = 0.7
 Architecture = auto
 
 # Pacman won't upgrade packages listed in IgnorePkg and members of IgnoreGroup
@@ -107,4 +106,3 @@ Include = /etc/pacman.d/mirrorlist
 #[custom]
 #SigLevel = Optional TrustAll
 #Server = file:///home/custompkgs
-

pacman-multilib-testing.conf

@@ -16,10 +16,9 @@
 #GPGDir      = /etc/pacman.d/gnupg/
 #HookDir     = /etc/pacman.d/hooks/
 HoldPkg     = pacman glibc
-#XferCommand = /usr/bin/curl -C - -f %u > %o
+#XferCommand = /usr/bin/curl -L -C - -f -o %o %u
 #XferCommand = /usr/bin/wget --passive-ftp -c -O %o %u
 #CleanMethod = KeepInstalled
-#UseDelta    = 0.7
 Architecture = auto
 
 # Pacman won't upgrade packages listed in IgnorePkg and members of IgnoreGroup
@@ -98,4 +97,3 @@ Include = /etc/pacman.d/mirrorlist
 #[custom]
 #SigLevel = Optional TrustAll
 #Server = file:///home/custompkgs
-

pacman-multilib.conf

@@ -16,10 +16,9 @@
 #GPGDir      = /etc/pacman.d/gnupg/
 #HookDir     = /etc/pacman.d/hooks/
 HoldPkg     = pacman glibc
-#XferCommand = /usr/bin/curl -C - -f %u > %o
+#XferCommand = /usr/bin/curl -L -C - -f -o %o %u
 #XferCommand = /usr/bin/wget --passive-ftp -c -O %o %u
 #CleanMethod = KeepInstalled
-#UseDelta    = 0.7
 Architecture = auto
 
 # Pacman won't upgrade packages listed in IgnorePkg and members of IgnoreGroup
@@ -99,4 +98,3 @@ Include = /etc/pacman.d/mirrorlist
 #[custom]
 #SigLevel = Optional TrustAll
 #Server = file:///home/custompkgs
-

pacman-staging.conf

@@ -16,10 +16,9 @@
 #GPGDir      = /etc/pacman.d/gnupg/
 #HookDir     = /etc/pacman.d/hooks/
 HoldPkg     = pacman glibc
-#XferCommand = /usr/bin/curl -C - -f %u > %o
+#XferCommand = /usr/bin/curl -L -C - -f -o %o %u
 #XferCommand = /usr/bin/wget --passive-ftp -c -O %o %u
 #CleanMethod = KeepInstalled
-#UseDelta    = 0.7
 Architecture = auto
 
 # Pacman won't upgrade packages listed in IgnorePkg and members of IgnoreGroup
@@ -96,4 +95,3 @@ Include = /etc/pacman.d/mirrorlist
 #[custom]
 #SigLevel = Optional TrustAll
 #Server = file:///home/custompkgs
-

pacman-testing.conf

@@ -16,10 +16,9 @@
 #GPGDir      = /etc/pacman.d/gnupg/
 #HookDir     = /etc/pacman.d/hooks/
 HoldPkg     = pacman glibc
-#XferCommand = /usr/bin/curl -C - -f %u > %o
+#XferCommand = /usr/bin/curl -L -C - -f -o %o %u
 #XferCommand = /usr/bin/wget --passive-ftp -c -O %o %u
 #CleanMethod = KeepInstalled
-#UseDelta    = 0.7
 Architecture = auto
 
 # Pacman won't upgrade packages listed in IgnorePkg and members of IgnoreGroup
@@ -90,4 +89,3 @@ Include = /etc/pacman.d/mirrorlist
 #[custom]
 #SigLevel = Optional TrustAll
 #Server = file:///home/custompkgs
-

zsh_completion.in

@@ -1,4 +1,4 @@
-#compdef archbuild archco arch-nspawn archrelease commitpkg finddeps makechrootpkg mkarchroot rebuildpkgs extrapkg=commitpkg corepkg=commitpkg testingpkg=commitpkg stagingpkg=commitpkg communitypkg=commitpkg community-testingpkg=commitpkg community-stagingpkg=commitpkg multilibpkg=commitpkg multilib-testingpkg=commitpkg extra-x86_64-build=archbuild testing-x86_64-build=archbuild staging-x86_64-build=archbuild multilib-build=archbuild multilib-testing-build=archbuild multilib-staging-build=archbuild kde-unstable-x86_64-build=archbuild gnome-unstable-x86_64-build=archbuild communityco=archco checkpkg sogrep offload-build
+#compdef archbuild archco arch-nspawn archrelease commitpkg finddeps makechrootpkg mkarchroot rebuildpkgs extrapkg=commitpkg corepkg=commitpkg testingpkg=commitpkg stagingpkg=commitpkg communitypkg=commitpkg community-testingpkg=commitpkg community-stagingpkg=commitpkg multilibpkg=commitpkg multilib-testingpkg=commitpkg extra-x86_64-build=archbuild testing-x86_64-build=archbuild staging-x86_64-build=archbuild multilib-build=archbuild multilib-testing-build=archbuild multilib-staging-build=archbuild kde-unstable-x86_64-build=archbuild gnome-unstable-x86_64-build=archbuild communityco=archco checkpkg sogrep offload-build makerepropkg
 # License: Unspecified
 
 m4_include(lib/valid-tags.sh)
@@ -58,6 +58,7 @@ _makechrootpkg_args=(
 )
 
 _mkarchroot_args=(
+	'-U[Install a package into the working copy]:target:_files -g "*.pkg.tar.*(.)"'
 	'-C[Location of a pacman config file]:pacman_config:_files -g "*.conf(.)"'
 	'-M[Location of a makepkg config file]:makepkg_config:_files -g "*.conf(.)"'
 	'-c[Set pacman cache]:pacman_cache:_files -/'
@@ -92,6 +93,14 @@ _offload_build_args=(
 	'(-h --help)'{-h,--help}'[Display usage]'
 )
 
+_makerepropkg_args=(
+	'-d[Run diffoscope if the package is unreproducible]'
+	'-c[Set pacman cache]:pacman_cache:_files -/'
+	'-M[Location of a makepkg config file]:makepkg_config:_files -g "*.conf(.)"'
+	'-h[Display usage]'
+	'*:working_dir:_files -g "*.pkg.tar.*(.)"'
+)
+
 _devtools_completions_all_packages() {
 	typeset -U packages
 	packages=($(_call_program packages pacman -Sql))