Canonicalize array initialization

* initialize cache_dirs with one of:
 1. value provided in command line -c switch
 2. pacman.conf provided in command line -C switch
 3. container pacman.conf file
* find first writable cache_dir, bind it rw, remove from cache_dirs
* extract host mirrors, find file:// ones, append to cache_dirs
* extract server url, find file:// ones, append to cache_dirs
* concatenate cache_dirs removing duplicates, bind them ro to container
* check if any extra cache_dirs was defined, append to
  container pacman.conf as new [options] section.

Fix #57 #58 #59

.gitignore

@@ -19,3 +19,4 @@ crossrepomove
 arch-nspawn
 sogrep
 doc/*.1
+doc/*.7

Makefile

@@ -1,4 +1,4 @@
-V=20200407
+V=20210202
 
 PREFIX = /usr/local
 MANDIR = $(PREFIX)/share/man
@@ -75,7 +75,8 @@ MANS = \
 	doc/makerepropkg.1 \
 	doc/mkarchroot.1 \
 	doc/find-libdeps.1 \
-	doc/find-libprovides.1
+	doc/find-libprovides.1 \
+	doc/devtools.7
 
 
 all: $(BINPROGS) bash_completion zsh_completion man

arch-nspawn.in

@@ -57,41 +57,51 @@ shift 1
 
 [[ -z $working_dir ]] && die 'Please specify a working directory.'
 
-pacconf_cmd=$(command -v pacman-conf || command -v pacconf)
-
 if (( ${#cache_dirs[@]} == 0 )); then
-	mapfile -t cache_dirs < <($pacconf_cmd --config "${pac_conf:-$working_dir/etc/pacman.conf}" CacheDir)
+	mapfile -t cache_dirs < <(pacman-conf --config "${pac_conf:-$working_dir/etc/pacman.conf}" CacheDir)
 fi
 
-# shellcheck disable=2016
-host_mirrors=($($pacconf_cmd --repo extra Server 2> /dev/null | sed -r 's#(.*/)extra/os/.*#\1$repo/os/$arch#'))
-
-for host_mirror in "${host_mirrors[@]}"; do
-	if [[ $host_mirror == *file://* ]]; then
-		host_mirror=$(echo "$host_mirror" | sed -r 's#file://(/.*)/\$repo/os/\$arch#\1#g')
-		for m in "$host_mirror"/pool/*/; do
-			in_array "$m" "${cache_dirs[@]}" || cache_dirs+=("$m")
-		done
+# find first writable cache, bind it to container in rw mode, remove it from cache_dirs
+for cache_dir in "${cache_dirs[@]}"; do
+	if [[ -d "$cache_dir" && -w "$cache_dir" ]]; then
+		mount_args+=("--bind=$cache_dir")
+		mapfile -t cache_dirs < <(printf "%s\n" "${cache_dirs[@]}" | grep -vFx "$cache_dir")
+		break
 	fi
+# this could print an warning to the user that no host caches would be used
 done
 
-while read -r line; do
-	mapfile -t lines < <($pacconf_cmd --config "${pac_conf:-$working_dir/etc/pacman.conf}" \
-		--repo $line Server | sed -r 's#(.*/)[^/]+/os/.+#\1#')
-	for line in "${lines[@]}"; do
-		if [[ $line = file://* ]]; then
-			line=${line#file://}
-			in_array "$line" "${cache_dirs[@]}" || cache_dirs+=("$line")
-		fi
+# shellcheck disable=2016
+mapfile -t host_mirrors < <(pacman-conf --repo extra Server 2> /dev/null | sed -r 's#(.*/)extra/os/.*#\1$repo/os/$arch#')
+
+# extract local host mirrors (Server=file://...)
+mapfile -t host_mirrors_local < <(printf "%s\n" "${host_mirrors[@]}" | grep -Po "(?<=file://).*(?=\/\$repo)")
+
+for host_mirror in "${host_mirrors_local[@]}"; do
+	for m in "$host_mirror"/pool/*/; do
+		cache_dirs+=("$m")
 	done
-done < <($pacconf_cmd --config "${pac_conf:-$working_dir/etc/pacman.conf}" --repo-list)
-
-mount_args+=("--bind=${cache_dirs[0]//:/\\:}")
-
-for cache_dir in "${cache_dirs[@]:1}"; do
-	mount_args+=("--bind-ro=${cache_dir//:/\\:}")
 done
 
+mapfile -t repos < <(pacman-conf --config "${pac_conf:-$working_dir/etc/pacman.conf}" --repo-list)
+
+declare -a servers
+for repo in "${repos[@]}"; do
+	mapfile -t -O "${#servers[@]}" servers < <( \
+		pacman-conf --config "${pac_conf:-$working_dir/etc/pacman.conf}" --repo "$repo" Server | \
+		grep -Po "(?<=file://).*$" | sed -r -e "s#\$repo#$repo#" -e 's#/$arch##' )
+done
+
+cache_dirs+=("${servers[@]}")
+
+# rest of cache_dirs are getting bind ro therefor we don't care about ordering and can use sort -u for concatenation
+if (( ${#cache_dirs[@]} != 0 )); then
+	mapfile -t cache_dirs < <(printf '%s\n' "${cache_dirs[@]}" | sort -u)
+	for cache_dir in "${cache_dirs[@]}"; do
+		mount_args+=("--bind-ro=${cache_dir//:/\\:}")
+	done
+fi
+
 # {{{ functions
 copy_hostconf () {
 	unshare --fork --pid gpg --homedir "$working_dir"/etc/pacman.d/gnupg/ --no-permission-warning --quiet --batch --import --import-options import-local-sigs "$(pacman-conf GpgDir)"/pubring.gpg >/dev/null 2>&1
@@ -102,13 +112,18 @@ copy_hostconf () {
 	[[ -n $pac_conf ]] && cp "$pac_conf" "$working_dir/etc/pacman.conf"
 	[[ -n $makepkg_conf ]] && cp "$makepkg_conf" "$working_dir/etc/makepkg.conf"
 
+	# check if any extra cache_dirs was added
+	mapfile -t container_cache_dirs < <(pacman-conf --config "$working_dir/etc/pacman.conf" CacheDir)
+	mapfile -t extra_cache_dirs < <(comm -13 <(printf "%s\n" "${container_cache_dirs[@]}"|sort) <(printf "%s\n" "${cache_dirs[@]}"|sort))
+	if (( ${#extra_cache_dirs[@]} != 0 )); then
+		echo -e "\n[options]\nCacheDir = ${extra_cache_dirs[*]}" >> "$working_dir/etc/pacman.conf"
+	fi
+
 	local file
 	for file in "${files[@]}"; do
 		mkdir -p "$(dirname "$working_dir$file")"
 		cp -T "$file" "$working_dir$file"
 	done
-
-	sed -r "s|^#?\\s*CacheDir.+|CacheDir = ${cache_dirs[*]}|g" -i "$working_dir/etc/pacman.conf"
 }
 # }}}
 

doc/devtools.7.asciidoc

@@ -0,0 +1,46 @@
+devtools(7)
+===========
+
+Name
+----
+devtools - Developer tools for the Arch Linux distribution
+
+Description
+-----------
+
+Devtools contains tools for package maintenance in Arch Linux. The toolset
+varies from tools for building packages in a clean chroot ('mkarchroot',...),
+packaging related tools for sonames ('sogrep', 'lddd') and tools for
+repository management such as ('archco', 'extra2community')
+
+Programs
+--------
+The list below gives a short overview; see the respective documentation
+for details.
+
+linkman:checkpkg[1]
+	Compare the current build pakcage with the repository version
+
+linkman:find-libdeps[1]
+	Find soname dependencies for a package
+
+linkman:find-libprovides[1]
+	Find soname's which are provided by a package
+
+linkman:lddd[1]
+	Find broken library links on your system
+
+linkman:makerepropkg[1]
+	Rebuild a package to see if it is reproducible
+
+linkman:mkarchroot[1]
+	Creates an arch chroot in a specified location with a specified set of
+	packages
+
+linkman:offload-build[1]
+	Build a PKGBUILD on a remote server using makechrootpkg
+
+linkman:sogrep[1]
+	Find packages using a linked to a given shared library
+
+include::footer.asciidoc[]

doc/makerepropkg.1.asciidoc

@@ -7,7 +7,7 @@ makerepropkg - Rebuild a package to see if it is reproducible
 
 Synopsis
 --------
-makerepropkg [OPTIONS] <package_file>...
+makerepropkg [OPTIONS] [<package_file|pkgname>...]
 
 Description
 -----------
@@ -24,6 +24,15 @@ When given multiple packages, additional package files are assumed to be split
 packages and will be treated as additional artifacts to compare during the
 verification step.
 
+A valid target(s) for pacman -S can be specified instead, and makerepropkg will
+download it to the cache if needed. This is mostly useful to specify which
+repository to retrieve from. If no positional arguments are specified, the
+targets will be sourced from the PKGBUILD.
+
+In either case, the package name will be converted to a filename from the
+cache, and makerepropkg will proceed as though this filename was initially
+specified.
+
 This implements a verifier for pacman/libalpm packages in accordance with the
 link:https://reproducible-builds.org/[Reproducible Builds] project.
 

doc/offload-build.1.asciidoc

@@ -28,7 +28,7 @@ Options
 	architecture officially supported by Arch Linux.
 
 *-s, --server* <hostname>::
-	Offload to a specific build server. The default is dragon.archlinux.org
+	Offload to a specific build server. The default is build.archlinux.org
 	which is used as part of the build toolchain for the official Arch Linux
 	repos.
 

makechrootpkg.in

@@ -242,6 +242,15 @@ download_sources() {
 		die "Could not download sources."
 }
 
+move_logfiles() {
+	local l
+	for l in "$copydir"/logdest/*; do
+		[[ $l == */logpipe.* ]] && continue
+		chown "$src_owner" "$l"
+		mv "$l" "$LOGDEST"
+	done
+}
+
 move_products() {
 	local pkgfile
 	for pkgfile in "$copydir"/pkgdest/*; do
@@ -254,12 +263,7 @@ move_products() {
 		fi
 	done
 
-	local l
-	for l in "$copydir"/logdest/*; do
-		[[ $l == */logpipe.* ]] && continue
-		chown "$src_owner" "$l"
-		mv "$l" "$LOGDEST"
-	done
+	move_logfiles
 
 	for s in "$copydir"/srcpkgdest/*; do
 		chown "$src_owner" "$s"
@@ -370,6 +374,7 @@ then
 	move_products
 else
 	(( ret += 1 ))
+	move_logfiles
 fi
 
 (( temp_chroot )) && delete_chroot "$copydir" "$copy"

makepkg-x86_64.conf

@@ -15,7 +15,7 @@ DLAGENTS=('file::/usr/bin/curl -gqC - -o %o %u'
           'ftp::/usr/bin/curl -gqfC - --ftp-pasv --retry 3 --retry-delay 3 -o %o %u'
           'http::/usr/bin/curl -gqb "" -fLC - --retry 3 --retry-delay 3 -o %o %u'
           'https::/usr/bin/curl -gqb "" -fLC - --retry 3 --retry-delay 3 -o %o %u'
-          'rsync::/usr/bin/rsync --no-motd -z %u %o'
+          'rsync::/usr/bin/rsync --no-motd -zz %u %o'
           'scp::/usr/bin/scp -C %u %o')
 
 # Other common tools:

makerepropkg.in

@@ -57,9 +57,15 @@ parse_buildinfo() {
 get_pkgfile() {
     local cdir=${cache_dirs[0]}
     local pkgfilebase=${1}
+    local mode=${2}
     local pkgname=${pkgfilebase%-*-*-*}
     local pkgfile ext
 
+    # try without downloading
+    if [[ ${mode} != localonly ]] && get_pkgfile "${pkgfilebase}" localonly; then
+        return 0
+    fi
+
     for ext in .zst .xz ''; do
         pkgfile=${pkgfilebase}.pkg.tar${ext}
 
@@ -72,6 +78,9 @@ get_pkgfile() {
 
         for f in "${pkgfile}" "${pkgfile}.sig"; do
             if [[ ! -f "${cdir}/${f}" ]]; then
+                if [[ ${mode} = localonly ]]; then
+                    continue 2
+                fi
                 msg2 "retrieving '%s'..." "${f}" >&2
                 curl -Llf -# -o "${cdir}/${f}" "${archiveurl}/${pkgname:0:1}/${pkgname}/${f}" || continue 2
             fi
@@ -115,20 +124,46 @@ shift $((OPTIND - 1))
 
 check_root
 
-if [[ -n $1 ]]; then
-    pkgfile="$1"
-    splitpkgs=("$@")
-    for f in "${splitpkgs[@]}"; do
-        if ! bsdtar -tqf "${f}" .BUILDINFO >/dev/null 2>&1; then
-            error "file is not a valid pacman package: '%s'" "${f}"
-            exit 1
-        fi
-    done
-else
-    error "no package file specified. Try '${BASH_SOURCE[0]##*/} -h' for more information. "
-    exit 1
+[[ -f PKGBUILD ]] || { error "No PKGBUILD in current directory."; exit 1; }
+
+# without arguments, get list of packages from PKGBUILD
+if [[ -z $1 ]]; then
+    mapfile -t pkgnames < <(source PKGBUILD; pacman -Sddp --print-format '%r/%n' "${pkgname[@]}")
+    wait $! || {
+        error "No package file specified and failed to retrieve package names from './PKGBUILD'."
+        plain "Try '${BASH_SOURCE[0]##*/} -h' for more information." >&2
+        exit 1
+    }
+    msg "Reproducing all pkgnames listed in ./PKGBUILD"
+    set -- "${pkgnames[@]}"
 fi
 
+# check each package to see if it's a file, and if not, try to download it
+# using pacman -Sw, and get the filename from there
+splitpkgs=()
+for p in "$@"; do
+    if [[ -f ${p} ]]; then
+        splitpkgs+=("${p}")
+    else
+        pkgfile_remote=$(pacman -Sddp "${p}" 2>/dev/null) || { error "package name '%s' not in repos" "${p}"; exit 1; }
+        pkgfile=${pkgfile_remote#file://}
+        if [[ ! -f ${pkgfile} ]]; then
+            msg "Downloading package '%s' into pacman's cache" "${pkgfile}"
+            sudo pacman -Swdd --noconfirm --logfile /dev/null "${p}" || exit 1
+            pkgfile_remote=$(pacman -Sddp "${p}" 2>/dev/null)
+            pkgfile="${pkgfile_remote#file://}"
+        fi
+        splitpkgs+=("${pkgfile}")
+    fi
+done
+
+for f in "${splitpkgs[@]}"; do
+    if ! bsdtar -tqf "${f}" .BUILDINFO >/dev/null 2>&1; then
+        error "file is not a valid pacman package: '%s'" "${f}"
+        exit 1
+    fi
+done
+
 if (( ${#cache_dirs[@]} == 0 )); then
 	mapfile -t cache_dirs < <(pacman-conf CacheDir)
 fi
@@ -139,11 +174,11 @@ load_makepkg_config
 HOME=${ORIG_HOME}
 [[ -d ${SRCDEST} ]] || SRCDEST=${PWD}
 
-parse_buildinfo < <(bsdtar -xOqf "${pkgfile}" .BUILDINFO)
+parse_buildinfo < <(bsdtar -xOqf "${splitpkgs[0]}" .BUILDINFO)
 export SOURCE_DATE_EPOCH="${buildinfo[builddate]}"
 PACKAGER="${buildinfo[packager]}"
 BUILDDIR="${buildinfo[builddir]}"
-PKGEXT=${pkgfile#${pkgfile%.pkg.tar*}}
+PKGEXT=${splitpkgs[0]#${splitpkgs[0]%.pkg.tar*}}
 
 # nuke and restore reproducible testenv
 for copy in "${buildroot}"/*/; do

offload-build.in

@@ -24,7 +24,7 @@ source /usr/share/makepkg/util/config.sh
 # global defaults suitable for use by Arch staff
 repo=extra
 arch=x86_64
-server=dragon.archlinux.org
+server=build.archlinux.org
 
 die() { printf "error: $1\n" "${@:2}"; exit 1; }
 
@@ -83,6 +83,9 @@ archbuild_cmd=("${repo}${archbuild_arch:+-$archbuild_arch}-build" "$@")
 
 trap 'rm -rf $TEMPDIR' EXIT INT TERM QUIT
 
+# Load makepkg.conf variables to be available
+load_makepkg_config
+
 # Use a source-only tarball as an intermediate to transfer files. This
 # guarantees the checksums are okay, and guarantees that all needed files are
 # transferred, including local sources, install scripts, and changelogs.
@@ -98,7 +101,7 @@ mapfile -t files < <(
     # This is sort of bash golfing but it allows running a mildly complex
     # command over ssh with a single connection.
     # shellcheck disable=SC2145
-    cat "$SRCPKGDEST"/*.src.tar.gz |
+    cat "$SRCPKGDEST"/*"$SRCEXT" |
         ssh $server '
             temp="${XDG_CACHE_HOME:-$HOME/.cache}/offload-build" &&
             mkdir -p "$temp" &&
@@ -122,7 +125,6 @@ mapfile -t files < <(
 
 if (( ${#files[@]} )); then
     printf '%s\n' '' '-> copying files...'
-    load_makepkg_config
     scp "${files[@]/#/$server:}" "${TEMPDIR}/"
     mv "${TEMPDIR}"/*.pkg.tar* "${PKGDEST:-${PWD}}/"
     mv "${TEMPDIR}/PKGBUILD" "${PWD}/"