Version 20191212

We need to modify the matching of valid package files to support formats
like zstd. Let's try to use an eager approach instead of a simple
whitelist in order to be functional for arbitrary formats that may be
introduced in the future without the need to adjust any code.

Allow any single fragment word as compression type but filter out known
non-package content like signature files.

Signed-off-by: Levente Polyak <anthraxx@archlinux.org>

.gitignore

@@ -9,10 +9,12 @@ commitpkg
 finddeps
 lddd
 makechrootpkg
+makerepropkg
 mkarchroot
 rebuildpkgs
 zsh_completion
 find-libdeps
 crossrepomove
 arch-nspawn
+sogrep
 doc/*.1

.travis.yml

@@ -4,7 +4,7 @@ sudo: required
 services:
 - docker
 
-arch:
+archlinux:
   packages:
     - openssh
     - subversion

Makefile

@@ -1,27 +1,28 @@
-V=20190912
+V=20191212
 
 PREFIX = /usr/local
 MANDIR = $(PREFIX)/share/man
 
 IN_PROGS = \
-	checkpkg \
-	commitpkg \
 	archco \
+	arch-nspawn \
 	archrelease \
 	archbuild \
-	lddd \
-	finddeps \
-	rebuildpkgs \
-	find-libdeps \
+	checkpkg \
+	commitpkg \
 	crossrepomove\
-	arch-nspawn \
+	finddeps \
+	find-libdeps \
+	lddd \
+	makerepropkg \
 	mkarchroot \
-	makechrootpkg
+	makechrootpkg \
+	rebuildpkgs \
+	sogrep
 
 BINPROGS = \
 	$(IN_PROGS) \
 	offload-build \
-	sogrep
 
 CONFIGFILES = \
 	makepkg-x86_64.conf \
@@ -71,6 +72,7 @@ MANS = \
 	doc/checkpkg.1 \
 	doc/offload-build.1 \
 	doc/sogrep.1 \
+	doc/makerepropkg.1 \
 	doc/mkarchroot.1 \
 	doc/find-libdeps.1 \
 	doc/find-libprovides.1
@@ -128,6 +130,12 @@ uninstall:
 		rm -f $(DESTDIR)$(MANDIR)/man$${manfile##*.}/$${manfile#doc/}; \
 	done;
 
+TODAY=$(shell date +"%Y%m%d")
+tag:
+	@sed -E "s|^V=[0-9]{8}|V=$(TODAY)|" -i Makefile
+	@git commit --gpg-sign --message "Version $(TODAY)" Makefile
+	@git tag --sign --message "Version $(TODAY)" $(TODAY)
+
 dist:
 	git archive --format=tar --prefix=devtools-$(V)/ $(V) | gzip -9 > devtools-$(V).tar.gz
 	gpg --detach-sign --use-agent devtools-$(V).tar.gz
@@ -138,5 +146,5 @@ upload:
 check: $(BINPROGS) bash_completion makepkg-x86_64.conf PKGBUILD.proto
 	shellcheck $^
 
-.PHONY: all clean install uninstall dist upload check
+.PHONY: all clean install uninstall dist upload check tag
 .DELETE_ON_ERROR:

arch-nspawn.in

@@ -20,6 +20,7 @@ umask 0022
 working_dir=''
 
 files=()
+mount_args=()
 
 usage() {
 	echo "Usage: ${0##*/} [options] working-dir [systemd-nspawn arguments]"
@@ -65,40 +66,35 @@ fi
 # shellcheck disable=2016
 host_mirrors=($($pacconf_cmd --repo extra Server 2> /dev/null | sed -r 's#(.*/)extra/os/.*#\1$repo/os/$arch#'))
 
+for host_mirror in "${host_mirrors[@]}"; do
+	if [[ $host_mirror == *file://* ]]; then
+		host_mirror=$(echo "$host_mirror" | sed -r 's#file://(/.*)/\$repo/os/\$arch#\1#g')
+		in_array "$host_mirror" "${cache_dirs[@]}" || cache_dirs+=("$host_mirror")
+	fi
+done
+
 while read -r line; do
 	mapfile -t lines < <($pacconf_cmd --config "${pac_conf:-$working_dir/etc/pacman.conf}" \
-		--repo $line Server | sed -r 's#(.*/)[^/]+/os/.+#\1$repo/os/$arch#')
-	if [[ ${lines[0]} != ${host_mirrors[0]} ]]; then
+		--repo $line Server | sed -r 's#(.*/)[^/]+/os/.+#\1#')
 	for line in "${lines[@]}"; do
 		if [[ $line = file://* ]]; then
 			line=${line#file://}
 			in_array "$line" "${cache_dirs[@]}" || cache_dirs+=("$line")
 		fi
 	done
-	fi
 done < <($pacconf_cmd --config "${pac_conf:-$working_dir/etc/pacman.conf}" --repo-list)
 
-# {{{ functions
-build_mount_args() {
-	declare -g mount_args=()
+mount_args+=("--bind=${cache_dirs[0]//:/\\:}")
 
-	for host_mirror in "${host_mirrors[@]}"; do
-		if [[ $host_mirror == *file://* ]]; then
-			host_mirror_path=$(echo "$host_mirror" | sed -r 's#file://(/.*)/\$repo/os/\$arch#\1#g')
-			mount_args+=("--bind-ro=${host_mirror_path//:/\\:}")
-		fi
-	done
-
-	mount_args+=("--bind=${cache_dirs[0]//:/\\:}")
-
-	for cache_dir in "${cache_dirs[@]:1}"; do
+for cache_dir in "${cache_dirs[@]:1}"; do
 	mount_args+=("--bind-ro=${cache_dir//:/\\:}")
-	done
-}
+done
 
+# {{{ functions
 copy_hostconf () {
 	unshare --fork --pid gpg --homedir "$working_dir"/etc/pacman.d/gnupg/ --no-permission-warning --quiet --batch --import --import-options import-local-sigs "$(pacman-conf GpgDir)"/pubring.gpg >/dev/null 2>&1
 	pacman-key --gpgdir "$working_dir"/etc/pacman.d/gnupg/ --import-trustdb "$(pacman-conf GpgDir)" >/dev/null 2>&1
+
 	printf 'Server = %s\n' "${host_mirrors[@]}" >"$working_dir/etc/pacman.d/mirrorlist"
 
 	[[ -n $pac_conf ]] && cp "$pac_conf" "$working_dir/etc/pacman.conf"
@@ -123,7 +119,6 @@ elif [[ $(cat "$working_dir/.arch-chroot") != "$CHROOT_VERSION" ]]; then
 	die "chroot '%s' is not at version %s. Please rebuild." "$working_dir" "$CHROOT_VERSION"
 fi
 
-build_mount_args
 copy_hostconf
 
 eval "$(grep -a '^CARCH=' "$working_dir/etc/makepkg.conf")"

archbuild.in

@@ -5,7 +5,7 @@ m4_include(lib/common.sh)
 m4_include(lib/archroot.sh)
 
 base_packages=(base-devel)
-makechrootpkg_args=(-c -n)
+makechrootpkg_args=(-c -n -C)
 
 cmd="${0##*/}"
 if [[ "${cmd%%-*}" == 'multilib' ]]; then
@@ -68,7 +68,7 @@ if ${clean_first} || [[ ! -d "${chroots}/${repo}-${arch}" ]]; then
 	lock_close 9
 
 	rm -rf --one-file-system "${chroots}/${repo}-${arch}"
-	mkdir -m755 -p "${chroots}/${repo}-${arch}"
+	(umask 0022; mkdir -p "${chroots}/${repo}-${arch}")
 	setarch "${arch}" mkarchroot \
 		-C "${pacman_config}" \
 		-M "${makepkg_config}" \

checkpkg.in

@@ -22,6 +22,53 @@ elif [[ -r "$HOME/.makepkg.conf" ]]; then
 	source "$HOME/.makepkg.conf"
 fi
 
+usage() {
+    cat <<- _EOF_
+		Usage: ${BASH_SOURCE[0]##*/} [OPTIONS]
+
+		Searches for a locally built package corresponding to the PKGBUILD, and
+		downloads the last version of that package from the Pacman repositories.
+		It then compares the list of .so files provided by each version of the
+		package and outputs if there are soname differences for the new package.
+		A directory is also created using mktemp with files containing a file
+		list for both packages and a library list for both packages.
+
+		OPTIONS
+		    -r, --rmdir     Remove the temporary directory
+		    -w, --warn      Print a warning in case of differences
+		    -h, --help      Show this help text
+_EOF_
+}
+
+RMDIR=0
+WARN=0
+
+OPT_SHORT='rwh'
+OPT_LONG=('rmdir' 'warn' 'help')
+if ! parseopts "$OPT_SHORT" "${OPT_LONG[@]}" -- "$@"; then
+    exit 1
+fi
+set  -- "${OPTRET[@]}"
+
+while :; do
+    case $1 in
+        -r|--rmdir)
+            RMDIR=1
+            ;;
+        -w|--warn)
+            WARN=1
+            ;;
+        -h|--help)
+            usage
+            exit 0
+            ;;
+        --)
+            shift; break
+            ;;
+    esac
+    shift
+done
+
 if [[ ! -f PKGBUILD ]]; then
 	die 'This must be run in the directory of a built package.'
 fi
@@ -33,6 +80,7 @@ if [[ ${arch[0]} == 'any' ]]; then
 fi
 
 STARTDIR=$(pwd)
+(( RMDIR )) && trap 'rm -rf $TEMPDIR' EXIT INT TERM QUIT
 TEMPDIR=$(mktemp -d --tmpdir checkpkg-script.XXXX)
 
 for _pkgname in "${pkgname[@]}"; do
@@ -85,11 +133,12 @@ for _pkgname in "${pkgname[@]}"; do
 	find-libprovides "$TEMPDIR/$oldpkg" 2>/dev/null | sort > "$TEMPDIR/libraries-$_pkgname-old"
 	find-libprovides "$pkgfile" 2>/dev/null | sort > "$TEMPDIR/libraries-$_pkgname"
 	if ! diff_output="$(sdiff -s "$TEMPDIR/libraries-$_pkgname-old" "$TEMPDIR/libraries-$_pkgname")"; then
-		msg "Sonames differ in $_pkgname!"
+		message="Sonames differ in $_pkgname!"
+		(( WARN )) && warning "$message" || msg "$message"
 		echo "$diff_output"
 	else
 		msg "No soname differences for %s." "$_pkgname"
 	fi
 done
 
-msg "Files saved to %s" "$TEMPDIR"
+(( RMDIR )) || msg "Files saved to %s" "$TEMPDIR"

commitpkg.in

@@ -83,7 +83,7 @@ while getopts ':l:a:s:f' flag; do
 done
 shift $(( OPTIND - 1 ))
 
-# check packages have the packager field set
+# check packages for validity
 for _arch in "${arch[@]}"; do
 	if [[ -n $commit_arch && ${_arch} != "$commit_arch" ]]; then
 		continue
@@ -91,10 +91,15 @@ for _arch in "${arch[@]}"; do
 	for _pkgname in "${pkgname[@]}"; do
 		fullver=$(get_full_version "$_pkgname")
 
-		if pkgfile=$(find_cached_package "$_pkgname" "$_arch" "$fullver"); then
+		if pkgfile=$(find_cached_package "$_pkgname" "$fullver" "$_arch"); then
 			if grep -q "packager = Unknown Packager" <(bsdtar -xOqf "$pkgfile" .PKGINFO); then
 				die "PACKAGER was not set when building package"
 			fi
+			hashsum=sha256sum
+			pkgbuild_hash=$(awk -v"hashsum=$hashsum" -F' = ' '$1 == "pkgbuild_"hashsum {print $2}' <(bsdtar -xOqf "$pkgfile" .BUILDINFO))
+			if [[ "$pkgbuild_hash" != "$($hashsum PKGBUILD|cut -d' ' -f1)" ]]; then
+				die "PKGBUILD $hashsum mismatch: expected $pkgbuild_hash"
+			fi
 		fi
 	done
 done
@@ -104,10 +109,10 @@ if [[ -z $server ]]; then
 fi
 
 if [[ -n $(svn status -q) ]]; then
-	msgtemplate="upgpkg: $pkgbase $(get_full_version)"$'\n\n'
+	msgtemplate="upgpkg: $pkgbase $(get_full_version)"
 	if [[ -n $1 ]]; then
 		stat_busy 'Committing changes to trunk'
-		svn commit -q -m "${msgtemplate}${1}" || die
+		svn commit -q -m "${msgtemplate}: ${1}" || die
 		stat_done
 	else
 		msgfile="$(mktemp)"
@@ -157,8 +162,8 @@ for _arch in "${arch[@]}"; do
 			fi
 			gpg --detach-sign --use-agent --no-armor "${SIGNWITHKEY[@]}" "${pkgfile}" || die
 		fi
-		if ! gpg --verify "$sigfile" >/dev/null 2>&1; then
-			die "Signature %s.sig is incorrect!" "$pkgfile"
+		if ! gpg --verify "$sigfile" "$pkgfile" >/dev/null 2>&1; then
+			die "Signature %s is incorrect!" "$sigfile"
 		fi
 		uploads+=("$sigfile")
 	done

doc/checkpkg.1.asciidoc

@@ -19,6 +19,19 @@ outputs if there are soname differences for the new package. A directory is
 also created using mktemp with files containing a file list for both packages
 and a library list for both packages.
 
+Options
+-------
+
+*-r, --rmdir*::
+	Remove the temporary directory created to contain the file and library list
+	of both packages.
+
+*-w, --warn*::
+	Print a warning instead of a regular message in case of soname differences.
+
+*-h, --help*::
+	Show a help text
+
 See Also
 --------
 

doc/makerepropkg.1.asciidoc

@@ -0,0 +1,38 @@
+makerepropkg(1)
+================
+
+Name
+----
+makerepropkg - Rebuild a package to see if it is reproducible
+
+Synopsis
+--------
+makerepropkg [OPTIONS] <package_file>
+
+Description
+-----------
+
+Given the path to a built pacman package, attempt to rebuild it using the
+PKGBUILD in the current directory. The package will be built in an environment
+as closely matching the environment of the initial package as possible, by
+building up a chroot to match the information exposed in the package's
+linkman:BUILDINFO[5] manifest. On success, the resulting package will be
+compared to the input package, and makerepropkg will report whether the
+artifacts are identical.
+
+This implements a verifier for pacman/libalpm packages in accordance with the
+link:https://reproducible-builds.org/[Reproducible Builds] project.
+
+Options
+-------
+
+*-c*::
+	Set the pacman cache directory.
+
+*-M* <file>::
+	Location of a makepkg config file.
+
+*-h*::
+	Show this usage message
+
+include::footer.asciidoc[]

doc/mkarchroot.1.asciidoc

@@ -20,6 +20,9 @@ en_US.UTF-8 and de_DE.UTF-8 locale and a generated machine-id.
 Options
 -------
 
+*-U*::
+	Use 'pacman -U' to install packages.
+
 *-C* <file>::
 	Location of a pacman config file.
 

lib/common.sh

@@ -17,7 +17,7 @@ export LANG=C
 shopt -s extglob
 
 # check if messages are to be printed using color
-if [[ -t 2 ]]; then
+if [[ -t 2 && "$TERM" != dumb ]]; then
 	colorize
 else
 	# shellcheck disable=2034
@@ -142,7 +142,7 @@ find_cached_package() {
 	for dir in "${searchdirs[@]}"; do
 		[[ -d $dir ]] || continue
 
-		for pkg in "$dir"/*.pkg.tar?(.?z); do
+		for pkg in "$dir"/*.pkg.tar?(.!(sig|*.*)); do
 			[[ -f $pkg ]] || continue
 
 			# avoid adding duplicates of the same inode
@@ -152,7 +152,7 @@ find_cached_package() {
 
 			# split apart package filename into parts
 			pkgbasename=${pkg##*/}
-			pkgbasename=${pkgbasename%.pkg.tar?(.?z)}
+			pkgbasename=${pkgbasename%.pkg.tar*}
 
 			arch=${pkgbasename##*-}
 			pkgbasename=${pkgbasename%-"$arch"}

lib/valid-repos.sh

@@ -0,0 +1,31 @@
+#!/hint/bash
+# License: Unspecified
+:
+
+# shellcheck disable=2034
+_repos=(
+	staging
+	testing
+	core
+	extra
+	community-staging
+	community-testing
+	community
+	multilib-staging
+	multilib-testing
+	multilib
+	gnome-unstable
+	kde-unstable
+)
+
+# shellcheck disable=2034
+_build_repos=(
+	staging
+	testing
+	extra
+	multilib-staging
+	multilib-testing
+	multilib
+	gnome-unstable
+	kde-unstable
+)

makechrootpkg.in

@@ -13,6 +13,8 @@
 m4_include(lib/common.sh)
 m4_include(lib/archroot.sh)
 
+source /usr/share/makepkg/util/config.sh
+
 shopt -s nullglob
 
 default_makepkg_args=(--syncdeps --noconfirm --log --holdver --skipinteg)
@@ -28,6 +30,7 @@ keepbuilddir=0
 update_first=0
 clean_first=0
 run_namcap=0
+run_checkpkg=0
 temp_chroot=0
 
 bindmounts_ro=()
@@ -72,32 +75,13 @@ usage() {
 	echo '           Useful for maintaining multiple copies'
 	echo "           Default: $copy"
 	echo '-n         Run namcap on the package'
+	echo '-C         Run checkpkg on the package'
 	echo '-T         Build in a temporary directory'
 	echo '-U         Run makepkg as a specified user'
 	exit 1
 }
 
 # {{{ functions
-# Usage: load_vars $makepkg_conf
-# Globals:
-#  - SRCDEST
-#  - SRCPKGDEST
-#  - PKGDEST
-#  - LOGDEST
-#  - MAKEFLAGS
-#  - PACKAGER
-load_vars() {
-	local makepkg_conf="$1" var
-
-	[[ -f $makepkg_conf ]] || return 1
-
-	for var in {SRC,SRCPKG,PKG,LOG}DEST MAKEFLAGS PACKAGER; do
-		[[ -z ${!var:-} ]] && eval "$(source "$makepkg_conf"; printf "%s='%s'" "$var" "${!var}")"
-	done
-
-	return 0
-}
-
 # Usage: sync_chroot $chrootdir $copydir [$copy]
 sync_chroot() {
 	local chrootdir=$1
@@ -289,7 +273,7 @@ move_products() {
 }
 # }}}
 
-while getopts 'hcur:I:l:nTD:d:U:' arg; do
+while getopts 'hcur:I:l:nCTD:d:U:' arg; do
 	case "$arg" in
 		c) clean_first=1 ;;
 		D) bindmounts_ro+=("--bind-ro=$OPTARG") ;;
@@ -299,6 +283,7 @@ while getopts 'hcur:I:l:nTD:d:U:' arg; do
 		I) install_pkgs+=("$OPTARG") ;;
 		l) copy="$OPTARG" ;;
 		n) run_namcap=1; makepkg_args+=(--install) ;;
+		C) run_checkpkg=1 ;;
 		T) temp_chroot=1; copy+="-$$" ;;
 		U) makepkg_user="$OPTARG" ;;
 		h|*) usage ;;
@@ -335,16 +320,12 @@ for arg in "${@:$OPTIND}"; do
 	esac
 done
 
-if [[ -n $SUDO_USER ]]; then
-	eval "USER_HOME=~$SUDO_USER"
-else
-	USER_HOME=$HOME
-fi
-
 umask 0022
 
-load_vars "${XDG_CONFIG_HOME:-$USER_HOME/.config}/pacman/makepkg.conf" || load_vars "$USER_HOME/.makepkg.conf"
-load_vars /etc/makepkg.conf
+ORIG_HOME=$HOME
+IFS=: read -r _ _ _ _ _ HOME _ < <(getent passwd "${SUDO_USER:-$USER}")
+load_makepkg_config
+HOME=$ORIG_HOME
 
 # Use PKGBUILD directory if these don't exist
 [[ -d $PKGDEST ]]    || PKGDEST=$PWD
@@ -385,6 +366,11 @@ if arch-nspawn "$copydir" \
 	"${bindmounts_ro[@]}" "${bindmounts_rw[@]}" \
 	/chrootbuild "${makepkg_args[@]}"
 then
+	pkgnames=()
+	for pkgfile in "$copydir"/pkgdest/*; do
+		pkgfile=${pkgfile##*/};
+		pkgnames+=("${pkgfile%-*-*-*}");
+	done
 	move_products
 else
 	(( ret += 1 ))
@@ -399,5 +385,32 @@ if (( ret != 0 )); then
 		die "Build failed, check %s/build" "$copydir"
 	fi
 else
+	if (( run_checkpkg )); then
+		msg "Running checkpkg"
+
+		# sync off-site databases for up-to-date queries
+		trap 'rm -rf $dbpath; cleanup' EXIT INT TERM QUIT
+		dbpath=$(mktemp -d --tmpdir makechrootpkg-database.XXXXXXXXXX)
+		mkdir -p "$dbpath"
+		pacman -Sy --dbpath "$dbpath" --logfile /dev/null
+
+		# query current package locations
+		remotepkgs=($(pacman -Sddp --dbpath "$dbpath" --logfile /dev/null "${pkgnames[@]}"))
+		if (( $? )); then
+			warning "Skipped checkpkg due to missing repo packages"
+			exit 0
+		fi
+
+		# download package files if any non-local location exists
+		for remotepkg in "${remotepkgs[@]}"; do
+			[[ $remotepkg == file://* ]] && continue
+			msg2 "Downloading current versions"
+			pacman --noconfirm -Swdd --dbpath "$dbpath" --logfile /dev/null "${pkgnames[@]}"
+			break
+		done
+
+		msg2 "Checking packages"
+		sudo -u "$makepkg_user" checkpkg --rmdir --warn
+	fi
 	true
 fi

makepkg-x86_64.conf

@@ -132,17 +132,17 @@ DBGSRCDIR="/usr/src/debug"
 COMPRESSGZ=(gzip -c -f -n)
 COMPRESSBZ2=(bzip2 -c -f)
 COMPRESSXZ=(xz -c -z -)
+COMPRESSZST=(zstd -c -z -q -)
 COMPRESSLRZ=(lrzip -q)
 COMPRESSLZO=(lzop -q)
 COMPRESSZ=(compress -c -f)
+COMPRESSLZ4=(lz4 -q)
+COMPRESSLZ=(lzip -c -f)
 
 #########################################################################
 # EXTENSION DEFAULTS
 #########################################################################
 #
-# WARNING: Do NOT modify these variables unless you know what you are
-#          doing.
-#
 PKGEXT='.pkg.tar.xz'
 SRCEXT='.src.tar.gz'
 

makerepropkg.in

@@ -0,0 +1,190 @@
+#!/bin/bash
+#   makerepropkg - rebuild a package to see if it is reproducible
+#
+#   Copyright (c) 2019 by Eli Schwartz <eschwartz@archlinux.org>
+#
+#   This program is free software; you can redistribute it and/or modify
+#   it under the terms of the GNU General Public License as published by
+#   the Free Software Foundation; either version 2 of the License, or
+#   (at your option) any later version.
+#
+#   This program is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#   GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License
+#   along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+
+m4_include(lib/common.sh)
+m4_include(lib/archroot.sh)
+
+source /usr/share/makepkg/util/config.sh
+source /usr/share/makepkg/util/message.sh
+
+declare -A buildinfo
+declare -a buildenv buildopts installed installpkgs
+
+archiveurl='https://archive.archlinux.org/packages'
+buildroot=/var/lib/archbuild/reproducible
+chroot=testenv
+
+parse_buildinfo() {
+    local line var val
+
+    while read -r line; do
+        var="${line%% = *}"
+        val="${line#* = }"
+        case ${var} in
+            buildenv)
+                buildenv+=("${val}")
+                ;;
+            options)
+                buildopts+=("${val}")
+                ;;
+            installed)
+                installed+=("${val}")
+                ;;
+            *)
+                buildinfo["${var}"]="${val}"
+                ;;
+        esac
+    done
+}
+
+get_pkgfile() {
+    local cdir=${cache_dirs[0]}
+    local pkgfilebase=${1}
+    local pkgname=${pkgfilebase%-*-*-*}
+    local pkgfile ext
+
+    for ext in .xz .zstd ''; do
+        pkgfile=${pkgfilebase}.pkg.tar${ext}
+
+        for c in "${cache_dirs[@]}"; do
+            if [[ -f ${c}/${pkgfile} ]]; then
+                cdir=${c}
+                break
+            fi
+        done
+
+        for f in "${pkgfile}" "${pkgfile}.sig"; do
+            if [[ ! -f "${cdir}/${f}" ]]; then
+                msg2 "retrieving '%s'..." "${f}" >&2
+                curl -Llf -# -o "${cdir}/${f}" "${archiveurl}/${pkgname:0:1}/${pkgname}/${f}" || continue 2
+            fi
+        done
+        printf '%s\n' "file://${cdir}/${pkgfile}"
+        return 0
+    done
+
+    return 1
+}
+
+usage() {
+    cat << __EOF__
+usage: ${BASH_SOURCE[0]##*/} [options] <package_file>
+
+Run this script in a PKGBUILD dir to build a package inside a
+clean chroot while attempting to reproduce it. The package file
+will be used to derive metadata needed for reproducing the
+package, including the .PKGINFO as well as the buildinfo.
+
+For more details see https://reproducible-builds.org/
+
+OPTIONS
+    -c <dir>      Set pacman cache
+    -M <file>     Location of a makepkg config file
+    -h            Show this usage message
+__EOF__
+}
+
+while getopts 'M:c:h' arg; do
+        case "$arg" in
+                M) archroot_args+=(-M "$OPTARG") ;;
+                c) cache_dirs+=("$OPTARG") ;;
+                h) usage; exit 0 ;;
+                *|?) usage; exit 1 ;;
+        esac
+done
+shift $((OPTIND - 1))
+
+check_root
+
+if [[ -n $1 ]]; then
+    pkgfile="$1"
+    if ! bsdtar -tqf "${pkgfile}" .BUILDINFO >/dev/null 2>&1; then
+        error "file is not a valid pacman package: '%s'" "${pkgfile}"
+        exit 1
+    fi
+else
+    error "no package file specified. Try '${BASH_SOURCE[0]##*/} -h' for more information. "
+    exit 1
+fi
+
+if (( ${#cache_dirs[@]} == 0 )); then
+	mapfile -t cache_dirs < <(pacman-conf CacheDir)
+fi
+
+ORIG_HOME=${HOME}
+IFS=: read -r _ _ _ _ _ HOME _ < <(getent passwd "${SUDO_USER:-$USER}")
+load_makepkg_config
+HOME=${ORIG_HOME}
+[[ -d ${SRCDEST} ]] || SRCDEST=${PWD}
+
+parse_buildinfo < <(bsdtar -xOqf "${pkgfile}" .BUILDINFO)
+export SOURCE_DATE_EPOCH="${buildinfo[builddate]}"
+PACKAGER="${buildinfo[packager]}"
+BUILDDIR="${buildinfo[builddir]}"
+
+# nuke and restore reproducible testenv
+for copy in "${buildroot}"/*/; do
+    [[ -d ${copy} ]] || continue
+    subvolume_delete_recursive "${copy}"
+done
+rm -rf --one-file-system "${buildroot}"
+(umask 0022; mkdir -p "${buildroot}")
+
+for fname in "${installed[@]}"; do
+    if ! allpkgfiles+=("$(get_pkgfile "${fname}")"); then
+        error "failed to retrieve ${fname}"
+        exit 1
+    fi
+done
+printf '%s\n' "${allpkgfiles[@]}" | mkarchroot -U "${archroot_args[@]}" "${buildroot}"/root - || exit 1
+
+
+# use makechrootpkg to prep the build directory
+makechrootpkg -r "${buildroot}" -l "${chroot}" -- --packagelist || exit 1
+
+# set detected makepkg.conf options
+{
+    for var in PACKAGER BUILDDIR; do
+        printf '%s=%s\n' "${var}" "${!var@Q}"
+    done
+    printf 'OPTIONS=(%s)\n' "${buildopts[*]@Q}"
+    printf 'BUILDENV=(%s)\n' "${buildenv[*]@Q}"
+} >> "${buildroot}/${chroot}"/etc/makepkg.conf >> "${buildroot}/${chroot}"/etc/makepkg.conf
+install -d -o "${SUDO_UID:-$UID}" -g "$(id -g "${SUDO_UID:-$UID}")" "${buildroot}/${chroot}/${BUILDDIR}"
+
+# kick off the build
+arch-nspawn "${buildroot}/${chroot}" \
+    --bind="${PWD}:/startdir" \
+    --bind="${SRCDEST}:/srcdest" \
+    /chrootbuild -C --noconfirm --log --holdver --skipinteg
+
+if (( $? == 0 )); then
+    msg2 "built succeeded! built packages can be found in ${buildroot}/${chroot}/pkgdest"
+    msg "comparing artifacts..."
+    if cmp -s "${pkgfile}" "${buildroot}/${chroot}/pkgdest/${pkgfile##*/}"; then
+        msg2 "Package successfully reproduced!"
+        exit 0
+    else
+        warning "Package is not reproducible. :("
+        sha256sum "${pkgfile}" "${buildroot}/${chroot}/pkgdest/${pkgfile##*/}"
+    fi
+fi
+
+# the package either failed to build, or was unreproducible
+exit 1

mkarchroot.in

@@ -18,6 +18,7 @@ m4_include(lib/archroot.sh)
 umask 0022
 
 working_dir=''
+umode=''
 
 files=()
 nspawn_args=()
@@ -25,6 +26,7 @@ nspawn_args=()
 usage() {
 	echo "Usage: ${0##*/} [options] working-dir package-list..."
 	echo ' options:'
+	echo '    -U            Use pacman -U to install packages'
 	echo '    -C <file>     Location of a pacman config file'
 	echo '    -M <file>     Location of a makepkg config file'
 	echo '    -c <dir>      Set pacman cache'
@@ -34,8 +36,9 @@ usage() {
 	exit 1
 }
 
-while getopts 'hC:M:c:f:s' arg; do
+while getopts 'hUC:M:c:f:s' arg; do
 	case "$arg" in
+		U) umode=U ;;
 		C) pac_conf="$OPTARG" ;;
 		M) makepkg_conf="$OPTARG" ;;
 		c) cache_dirs+=("$OPTARG") ;;
@@ -44,8 +47,10 @@ while getopts 'hC:M:c:f:s' arg; do
 		h|?) usage ;;
 		*) error "invalid argument '%s'" "$arg"; usage ;;
 	esac
+	if [[ $arg != U ]]; then
 		nspawn_args+=("-$arg")
 		[[ -v OPTARG ]] && nspawn_args+=("$OPTARG")
+	fi
 done
 shift $((OPTIND - 1))
 
@@ -85,7 +90,7 @@ for file in "${files[@]}"; do
 	cp "$file" "$working_dir$file"
 done
 
-pacstrap -Mcd ${pac_conf:+-C "$pac_conf"} "$working_dir" \
+pacstrap -${umode}Mcd ${pac_conf:+-C "$pac_conf"} "$working_dir" \
 	"${cache_dirs[@]/#/--cachedir=}" "$@" || die 'Failed to install all packages'
 
 printf '%s.UTF-8 UTF-8\n' en_US de_DE > "$working_dir/etc/locale.gen"

offload-build

@@ -18,6 +18,8 @@
 #   along with this program.  If not, see <https://www.gnu.org/licenses/>.
 #
 
+source /usr/share/makepkg/util/config.sh
+
 
 # global defaults suitable for use by Arch staff
 repo=extra
@@ -78,7 +80,7 @@ fi
 
 archbuild_cmd=("${repo}${arch:+-$arch}-build" "$@")
 
-trap 'rm -rf $SRCPKGDEST' EXIT
+trap 'rm -rf $SRCPKGDEST' EXIT INT TERM QUIT
 
 # Use a source-only tarball as an intermediate to transfer files. This
 # guarantees the checksums are okay, and guarantees that all needed files are
@@ -86,6 +88,10 @@ trap 'rm -rf $SRCPKGDEST' EXIT
 export SRCPKGDEST=$(mktemp -d)
 makepkg --source || die "unable to make source package"
 
+# Temporary cosmetic workaround makepkg if SRCDEST is set somewhere else
+# but an empty src dir is created in PWD. Remove once fixed in makepkg.
+rmdir --ignore-fail-on-non-empty src 2>/dev/null || true
+
 mapfile -t files < <(
     # This is sort of bash golfing but it allows running a mildly complex
     # command over ssh with a single connection.
@@ -105,4 +111,11 @@ mapfile -t files < <(
             makepkg --packagelist
 ')
 
-(( ${#files[@]} )) && printf '%s\n' '' '-> copying files...' && scp "${files[@]/#/$server:}" .
+
+if (( ${#files[@]} )); then
+    printf '%s\n' '' '-> copying files...'
+    load_makepkg_config
+    scp "${files[@]/#/$server:}" "${PKGDEST:-${PWD}}/"
+else
+    exit 1
+fi

pacman-extra.conf

@@ -16,10 +16,9 @@
 #GPGDir      = /etc/pacman.d/gnupg/
 #HookDir     = /etc/pacman.d/hooks/
 HoldPkg     = pacman glibc
-#XferCommand = /usr/bin/curl -C - -f %u > %o
+#XferCommand = /usr/bin/curl -L -C - -f -o %o %u
 #XferCommand = /usr/bin/wget --passive-ftp -c -O %o %u
 #CleanMethod = KeepInstalled
-#UseDelta    = 0.7
 Architecture = auto
 
 # Pacman won't upgrade packages listed in IgnorePkg and members of IgnoreGroup
@@ -90,4 +89,3 @@ Include = /etc/pacman.d/mirrorlist
 #[custom]
 #SigLevel = Optional TrustAll
 #Server = file:///home/custompkgs
-

pacman-gnome-unstable.conf

@@ -16,10 +16,9 @@
 #GPGDir      = /etc/pacman.d/gnupg/
 #HookDir     = /etc/pacman.d/hooks/
 HoldPkg     = pacman glibc
-#XferCommand = /usr/bin/curl -C - -f %u > %o
+#XferCommand = /usr/bin/curl -L -C - -f -o %o %u
 #XferCommand = /usr/bin/wget --passive-ftp -c -O %o %u
 #CleanMethod = KeepInstalled
-#UseDelta    = 0.7
 Architecture = auto
 
 # Pacman won't upgrade packages listed in IgnorePkg and members of IgnoreGroup
@@ -99,4 +98,3 @@ Include = /etc/pacman.d/mirrorlist
 #[custom]
 #SigLevel = Optional TrustAll
 #Server = file:///home/custompkgs
-

pacman-kde-unstable.conf

@@ -16,10 +16,9 @@
 #GPGDir      = /etc/pacman.d/gnupg/
 #HookDir     = /etc/pacman.d/hooks/
 HoldPkg     = pacman glibc
-#XferCommand = /usr/bin/curl -C - -f %u > %o
+#XferCommand = /usr/bin/curl -L -C - -f -o %o %u
 #XferCommand = /usr/bin/wget --passive-ftp -c -O %o %u
 #CleanMethod = KeepInstalled
-#UseDelta    = 0.7
 Architecture = auto
 
 # Pacman won't upgrade packages listed in IgnorePkg and members of IgnoreGroup
@@ -93,4 +92,3 @@ Include = /etc/pacman.d/mirrorlist
 #[custom]
 #SigLevel = Optional TrustAll
 #Server = file:///home/custompkgs
-

pacman-multilib-staging.conf

@@ -16,10 +16,9 @@
 #GPGDir      = /etc/pacman.d/gnupg/
 #HookDir     = /etc/pacman.d/hooks/
 HoldPkg     = pacman glibc
-#XferCommand = /usr/bin/curl -C - -f %u > %o
+#XferCommand = /usr/bin/curl -L -C - -f -o %o %u
 #XferCommand = /usr/bin/wget --passive-ftp -c -O %o %u
 #CleanMethod = KeepInstalled
-#UseDelta    = 0.7
 Architecture = auto
 
 # Pacman won't upgrade packages listed in IgnorePkg and members of IgnoreGroup
@@ -107,4 +106,3 @@ Include = /etc/pacman.d/mirrorlist
 #[custom]
 #SigLevel = Optional TrustAll
 #Server = file:///home/custompkgs
-

pacman-multilib-testing.conf

@@ -16,10 +16,9 @@
 #GPGDir      = /etc/pacman.d/gnupg/
 #HookDir     = /etc/pacman.d/hooks/
 HoldPkg     = pacman glibc
-#XferCommand = /usr/bin/curl -C - -f %u > %o
+#XferCommand = /usr/bin/curl -L -C - -f -o %o %u
 #XferCommand = /usr/bin/wget --passive-ftp -c -O %o %u
 #CleanMethod = KeepInstalled
-#UseDelta    = 0.7
 Architecture = auto
 
 # Pacman won't upgrade packages listed in IgnorePkg and members of IgnoreGroup
@@ -98,4 +97,3 @@ Include = /etc/pacman.d/mirrorlist
 #[custom]
 #SigLevel = Optional TrustAll
 #Server = file:///home/custompkgs
-

pacman-multilib.conf

@@ -16,10 +16,9 @@
 #GPGDir      = /etc/pacman.d/gnupg/
 #HookDir     = /etc/pacman.d/hooks/
 HoldPkg     = pacman glibc
-#XferCommand = /usr/bin/curl -C - -f %u > %o
+#XferCommand = /usr/bin/curl -L -C - -f -o %o %u
 #XferCommand = /usr/bin/wget --passive-ftp -c -O %o %u
 #CleanMethod = KeepInstalled
-#UseDelta    = 0.7
 Architecture = auto
 
 # Pacman won't upgrade packages listed in IgnorePkg and members of IgnoreGroup
@@ -99,4 +98,3 @@ Include = /etc/pacman.d/mirrorlist
 #[custom]
 #SigLevel = Optional TrustAll
 #Server = file:///home/custompkgs
-

pacman-staging.conf

@@ -16,10 +16,9 @@
 #GPGDir      = /etc/pacman.d/gnupg/
 #HookDir     = /etc/pacman.d/hooks/
 HoldPkg     = pacman glibc
-#XferCommand = /usr/bin/curl -C - -f %u > %o
+#XferCommand = /usr/bin/curl -L -C - -f -o %o %u
 #XferCommand = /usr/bin/wget --passive-ftp -c -O %o %u
 #CleanMethod = KeepInstalled
-#UseDelta    = 0.7
 Architecture = auto
 
 # Pacman won't upgrade packages listed in IgnorePkg and members of IgnoreGroup
@@ -96,4 +95,3 @@ Include = /etc/pacman.d/mirrorlist
 #[custom]
 #SigLevel = Optional TrustAll
 #Server = file:///home/custompkgs
-

pacman-testing.conf

@@ -16,10 +16,9 @@
 #GPGDir      = /etc/pacman.d/gnupg/
 #HookDir     = /etc/pacman.d/hooks/
 HoldPkg     = pacman glibc
-#XferCommand = /usr/bin/curl -C - -f %u > %o
+#XferCommand = /usr/bin/curl -L -C - -f -o %o %u
 #XferCommand = /usr/bin/wget --passive-ftp -c -O %o %u
 #CleanMethod = KeepInstalled
-#UseDelta    = 0.7
 Architecture = auto
 
 # Pacman won't upgrade packages listed in IgnorePkg and members of IgnoreGroup
@@ -90,4 +89,3 @@ Include = /etc/pacman.d/mirrorlist
 #[custom]
 #SigLevel = Optional TrustAll
 #Server = file:///home/custompkgs
-

sogrep.in

@@ -21,10 +21,8 @@
 # globals
 : ${SOLINKS_MIRROR:="https://mirror.pkgbuild.com"}
 : ${SOCACHE_DIR:="${XDG_CACHE_HOME:-${HOME}/.cache}/sogrep"}
-repos=('staging' 'testing' 'core' 'extra'
-       'community-staging' 'community-testing' 'community'
-       'multilib-staging' 'multilib-testing' 'multilib'
-       'gnome-unstable' 'kde-unstable')
+
+m4_include(lib/valid-repos.sh)
 arches=('x86_64')
 
 # options
@@ -39,20 +37,20 @@ recache() {
 
     (( VERBOSE )) && verbosity=--progress-bar
 
-    for repo in "${repos[@]}"; do
+    for repo in "${_repos[@]}"; do
         for arch in "${arches[@]}"; do
             rm -rf "${SOCACHE_DIR}/${arch}/${repo}"
             mkdir -p "${SOCACHE_DIR}/${arch}/${repo}"
-            curl "$verbosity" "${SOLINKS_MIRROR}/${repo}/os/${arch}/${repo}.links.tar.gz" | bsdtar -xf - -C "${SOCACHE_DIR}/${arch}/${repo}"
+            curl -L "$verbosity" "${SOLINKS_MIRROR}/${repo}/os/${arch}/${repo}.links.tar.gz" | bsdtar -xf - -C "${SOCACHE_DIR}/${arch}/${repo}"
         done
     done
 }
 
 search() {
-    local repo=$1 arch lib=$2 srepos=("${repos[@]}")
+    local repo=$1 arch lib=$2 srepos=("${_repos[@]}")
 
     if [[ $repo != all ]]; then
-        if ! in_array "${repo}" "${repos[@]}"; then
+        if ! in_array "${repo}" "${_repos[@]}"; then
             echo "${BASH_SOURCE[0]##*/}: unrecognized repo '$repo'"
             echo "Try '${BASH_SOURCE[0]##*/} --help' for more information."
             exit 1

zsh_completion.in

@@ -1,11 +1,15 @@
-#compdef archbuild archco arch-nspawn archrelease commitpkg finddeps makechrootpkg mkarchroot rebuildpkgs extrapkg=commitpkg corepkg=commitpkg testingpkg=commitpkg stagingpkg=commitpkg communitypkg=commitpkg community-testingpkg=commitpkg community-stagingpkg=commitpkg multilibpkg=commitpkg multilib-testingpkg=commitpkg extra-x86_64-build=archbuild testing-x86_64-build=archbuild staging-x86_64-build=archbuild multilib-build=archbuild multilib-testing-build=archbuild multilib-staging-build=archbuild kde-unstable-x86_64-build=archbuild gnome-unstable-x86_64-build=archbuild communityco=archco
+#compdef archbuild archco arch-nspawn archrelease commitpkg finddeps makechrootpkg mkarchroot rebuildpkgs extrapkg=commitpkg corepkg=commitpkg testingpkg=commitpkg stagingpkg=commitpkg communitypkg=commitpkg community-testingpkg=commitpkg community-stagingpkg=commitpkg multilibpkg=commitpkg multilib-testingpkg=commitpkg extra-x86_64-build=archbuild testing-x86_64-build=archbuild staging-x86_64-build=archbuild multilib-build=archbuild multilib-testing-build=archbuild multilib-staging-build=archbuild kde-unstable-x86_64-build=archbuild gnome-unstable-x86_64-build=archbuild communityco=archco checkpkg sogrep offload-build makerepropkg
 # License: Unspecified
 
 m4_include(lib/valid-tags.sh)
+m4_include(lib/valid-repos.sh)
+
+_binary_arch=${_arch[*]:0:-1}
 
 _archbuild_args=(
 	'-c[Recreate the chroot before building]'
 	'-r[Create chroots in this directory]:base_dir:_files -/'
+	'-h[Display usage]'
 )
 
 _archco_args=(
@@ -13,19 +17,25 @@ _archco_args=(
 )
 
 _arch_nspawn_args=(
-	'-C[Location of a pacman config file]:pacman_config:_files'
-	'-M[Location of a makepkg config file]:makepkg_config:_files'
+	'-C[Location of a pacman config file]:pacman_config:_files -g "*.conf(.)"'
+	'-M[Location of a makepkg config file]:makepkg_config:_files -g "*.conf(.)"'
 	'-c[Set pacman cache]:pacman_cache:_files -/'
+	'-f[Copy file from the host to the chroot]:copy_file:_files'
+	'-s[Do not run setarch]'
 	'-h[Display usage]'
+	'1:chroot_dir:_files -/'
 )
 
 _archrelease_args=(
+	'-f[Force release without checks]'
 	"*:arch:($_tags[*])"
 )
 
 _commitpkg_args=(
-	"-a[Release to a specific architecture only]:arch:($_arch[*])"
+	'-f[Force release without checks]'
+	'-s[Target repo server]'
 	'-l[Set bandwidth limit]:limit'
+	"-a[Release to a specific architecture only]:arch:($_arch[*])"
 	'1:commit_msg'
 )
 
@@ -34,19 +44,27 @@ _finddeps_args=(
 )
 
 _makechrootpkg_args=(
-	'-I[Install a package into the working copy]:target:_files -g "*.pkg.tar.*(.)"'
-	'-c[Clean the chroot before building]'
 	'-h[Display usage]'
-	'-l[The directory to use as the working copy]:copy_dir:_files -/'
-	'-r[The chroot dir to use]:chroot_dir:_files -/'
+	'-c[Clean the chroot before building]'
+	'-d[Bind directory into build chroot as read-write]:bind_dir_rw:_files -/'
+	'-D[Bind directory into build chroot as read-only]:bind_dir_ro:_files -/'
 	'-u[Update the working copy of the chroot before building]'
+	'-r[The chroot dir to use]:chroot_dir:_files -/'
+	'-I[Install a package into the working copy]:target:_files -g "*.pkg.tar.*(.)"'
+	'-l[The directory to use as the working copy]:copy_dir:_files -/'
+	'-n[Run namcap on the package]'
+	'-T[Build in a temporary directory]'
+	'-U[Run makepkg as a specified user]:makepkg_user'
 )
 
 _mkarchroot_args=(
-	'-C[Location of a pacman config file]:pacman_config:_files'
-	'-M[Location of a makepkg config file]:makepkg_config:_files'
+	'-U[Install a package into the working copy]:target:_files -g "*.pkg.tar.*(.)"'
+	'-C[Location of a pacman config file]:pacman_config:_files -g "*.conf(.)"'
+	'-M[Location of a makepkg config file]:makepkg_config:_files -g "*.conf(.)"'
 	'-c[Set pacman cache]:pacman_cache:_files -/'
 	'-h[Display usage]'
+	'1:working_dir:_files -/'
+	'*:packages:_devtools_completions_all_packages'
 )
 
 _rebuildpkgs_args=(
@@ -54,6 +72,34 @@ _rebuildpkgs_args=(
 	'*:packages:_devtools_completions_all_packages'
 )
 
+_checkpkg_args=(
+	'(-r --rmdir)'{-r,--rmdir}'[Remove the temporary directory]'
+	'(-w --warn)'{-w,--warn}'[Print a warning in case of differences]'
+	'(-h --help)'{-h,--help}'[Display usage]'
+)
+
+_sogrep_args=(
+	'(-v --verbose)'{-v,--verbose}'[Show matched links in addition to pkgname]'
+	'(-r --refresh)'{-r,--refresh}'[Refresh the links databases]'
+	'(-h --help)'{-h,--help}'[Display usage]'
+	'1:repo:(all $_repos[*])'
+	'2:libname'
+)
+
+_offload_build_args=(
+	'(-r --repo)'{-r,--repo}'[Build against a specific repository]:repo:($_build_repos[*])'
+	'(-a --arch)'{-a,--arch}'[Build against a specific architecture]:arch:(${_binary_arch[*]})'
+	'(-s --server)'{-s,--server}'[Offload to a specific Build server]:server:'
+	'(-h --help)'{-h,--help}'[Display usage]'
+)
+
+_makerepropkg_args=(
+	'-c[Set pacman cache]:pacman_cache:_files -/'
+	'-M[Location of a makepkg config file]:makepkg_config:_files -g "*.conf(.)"'
+	'-h[Display usage]'
+	'1:working_dir:_files -g "*.pkg.tar.*(.)"'
+)
+
 _devtools_completions_all_packages() {
 	typeset -U packages
 	packages=($(_call_program packages pacman -Sql))