arch-nspawn: generalize mirrorlist variable replacement

The host_mirrors value uses a sed pattern that is closely coupled with
the mirror URL pattern in-use by most Arch mirrors. For some variants,
like Arch Linux ARM, values such as $repo and $arch sit in other parts
in the mirror path. This change makes the sed replacement more generic
to accomodate for variations in mirrorlist URLs while remaining backward
compatible.

.gitignore

@@ -19,4 +19,3 @@ crossrepomove
 arch-nspawn
 sogrep
 doc/*.1
-doc/*.7

Makefile

@@ -1,4 +1,4 @@
-V=20210202
+V=20200407
 
 PREFIX = /usr/local
 MANDIR = $(PREFIX)/share/man
@@ -75,8 +75,7 @@ MANS = \
 	doc/makerepropkg.1 \
 	doc/mkarchroot.1 \
 	doc/find-libdeps.1 \
-	doc/find-libprovides.1 \
-	doc/devtools.7
+	doc/find-libprovides.1
 
 
 all: $(BINPROGS) bash_completion zsh_completion man

arch-nspawn.in

@@ -58,17 +58,18 @@ shift 1
 [[ -z $working_dir ]] && die 'Please specify a working directory.'
 
 pacconf_cmd=$(command -v pacman-conf || command -v pacconf)
+pacconf_arch=$($pacconf_cmd architecture)
 
 if (( ${#cache_dirs[@]} == 0 )); then
 	mapfile -t cache_dirs < <($pacconf_cmd --config "${pac_conf:-$working_dir/etc/pacman.conf}" CacheDir)
 fi
 
 # shellcheck disable=2016
-host_mirrors=($($pacconf_cmd --repo extra Server 2> /dev/null | sed -r 's#(.*/)extra/os/.*#\1$repo/os/$arch#'))
+host_mirrors=($($pacconf_cmd --repo extra Server 2> /dev/null | sed -r 's#'"${pacconf_arch}"'#$arch#;s#extra#$repo#'))
 
 for host_mirror in "${host_mirrors[@]}"; do
 	if [[ $host_mirror == *file://* ]]; then
-		host_mirror=$(echo "$host_mirror" | sed -r 's#file://(/.*)/\$repo/os/\$arch#\1#g')
+		host_mirror=$(echo "$host_mirror" | sed -r 's#file://(/.*)/\$repo[/.]*#\1#g')
 		for m in "$host_mirror"/pool/*/; do
 			in_array "$m" "${cache_dirs[@]}" || cache_dirs+=("$m")
 		done

doc/devtools.7.asciidoc

@@ -1,46 +0,0 @@
-devtools(7)
-===========
-
-Name
-----
-devtools - Developer tools for the Arch Linux distribution
-
-Description
------------
-
-Devtools contains tools for package maintenance in Arch Linux. The toolset
-varies from tools for building packages in a clean chroot ('mkarchroot',...),
-packaging related tools for sonames ('sogrep', 'lddd') and tools for
-repository management such as ('archco', 'extra2community')
-
-Programs
---------
-The list below gives a short overview; see the respective documentation
-for details.
-
-linkman:checkpkg[1]
-	Compare the current build pakcage with the repository version
-
-linkman:find-libdeps[1]
-	Find soname dependencies for a package
-
-linkman:find-libprovides[1]
-	Find soname's which are provided by a package
-
-linkman:lddd[1]
-	Find broken library links on your system
-
-linkman:makerepropkg[1]
-	Rebuild a package to see if it is reproducible
-
-linkman:mkarchroot[1]
-	Creates an arch chroot in a specified location with a specified set of
-	packages
-
-linkman:offload-build[1]
-	Build a PKGBUILD on a remote server using makechrootpkg
-
-linkman:sogrep[1]
-	Find packages using a linked to a given shared library
-
-include::footer.asciidoc[]

doc/makerepropkg.1.asciidoc

@@ -7,7 +7,7 @@ makerepropkg - Rebuild a package to see if it is reproducible
 
 Synopsis
 --------
-makerepropkg [OPTIONS] [<package_file|pkgname>...]
+makerepropkg [OPTIONS] <package_file>...
 
 Description
 -----------
@@ -24,15 +24,6 @@ When given multiple packages, additional package files are assumed to be split
 packages and will be treated as additional artifacts to compare during the
 verification step.
 
-A valid target(s) for pacman -S can be specified instead, and makerepropkg will
-download it to the cache if needed. This is mostly useful to specify which
-repository to retrieve from. If no positional arguments are specified, the
-targets will be sourced from the PKGBUILD.
-
-In either case, the package name will be converted to a filename from the
-cache, and makerepropkg will proceed as though this filename was initially
-specified.
-
 This implements a verifier for pacman/libalpm packages in accordance with the
 link:https://reproducible-builds.org/[Reproducible Builds] project.
 

doc/offload-build.1.asciidoc

@@ -28,7 +28,7 @@ Options
 	architecture officially supported by Arch Linux.
 
 *-s, --server* <hostname>::
-	Offload to a specific build server. The default is build.archlinux.org
+	Offload to a specific build server. The default is dragon.archlinux.org
 	which is used as part of the build toolchain for the official Arch Linux
 	repos.
 

makechrootpkg.in

@@ -242,15 +242,6 @@ download_sources() {
 		die "Could not download sources."
 }
 
-move_logfiles() {
-	local l
-	for l in "$copydir"/logdest/*; do
-		[[ $l == */logpipe.* ]] && continue
-		chown "$src_owner" "$l"
-		mv "$l" "$LOGDEST"
-	done
-}
-
 move_products() {
 	local pkgfile
 	for pkgfile in "$copydir"/pkgdest/*; do
@@ -263,7 +254,12 @@ move_products() {
 		fi
 	done
 
-	move_logfiles
+	local l
+	for l in "$copydir"/logdest/*; do
+		[[ $l == */logpipe.* ]] && continue
+		chown "$src_owner" "$l"
+		mv "$l" "$LOGDEST"
+	done
 
 	for s in "$copydir"/srcpkgdest/*; do
 		chown "$src_owner" "$s"
@@ -374,7 +370,6 @@ then
 	move_products
 else
 	(( ret += 1 ))
-	move_logfiles
 fi
 
 (( temp_chroot )) && delete_chroot "$copydir" "$copy"

makepkg-x86_64.conf

@@ -15,7 +15,7 @@ DLAGENTS=('file::/usr/bin/curl -gqC - -o %o %u'
           'ftp::/usr/bin/curl -gqfC - --ftp-pasv --retry 3 --retry-delay 3 -o %o %u'
           'http::/usr/bin/curl -gqb "" -fLC - --retry 3 --retry-delay 3 -o %o %u'
           'https::/usr/bin/curl -gqb "" -fLC - --retry 3 --retry-delay 3 -o %o %u'
-          'rsync::/usr/bin/rsync --no-motd -zz %u %o'
+          'rsync::/usr/bin/rsync --no-motd -z %u %o'
           'scp::/usr/bin/scp -C %u %o')
 
 # Other common tools:

makerepropkg.in

@@ -57,15 +57,9 @@ parse_buildinfo() {
 get_pkgfile() {
     local cdir=${cache_dirs[0]}
     local pkgfilebase=${1}
-    local mode=${2}
     local pkgname=${pkgfilebase%-*-*-*}
     local pkgfile ext
 
-    # try without downloading
-    if [[ ${mode} != localonly ]] && get_pkgfile "${pkgfilebase}" localonly; then
-        return 0
-    fi
-
     for ext in .zst .xz ''; do
         pkgfile=${pkgfilebase}.pkg.tar${ext}
 
@@ -78,9 +72,6 @@ get_pkgfile() {
 
         for f in "${pkgfile}" "${pkgfile}.sig"; do
             if [[ ! -f "${cdir}/${f}" ]]; then
-                if [[ ${mode} = localonly ]]; then
-                    continue 2
-                fi
                 msg2 "retrieving '%s'..." "${f}" >&2
                 curl -Llf -# -o "${cdir}/${f}" "${archiveurl}/${pkgname:0:1}/${pkgname}/${f}" || continue 2
             fi
@@ -124,45 +115,19 @@ shift $((OPTIND - 1))
 
 check_root
 
-[[ -f PKGBUILD ]] || { error "No PKGBUILD in current directory."; exit 1; }
-
-# without arguments, get list of packages from PKGBUILD
-if [[ -z $1 ]]; then
-    mapfile -t pkgnames < <(source PKGBUILD; pacman -Sddp --print-format '%r/%n' "${pkgname[@]}")
-    wait $! || {
-        error "No package file specified and failed to retrieve package names from './PKGBUILD'."
-        plain "Try '${BASH_SOURCE[0]##*/} -h' for more information." >&2
-        exit 1
-    }
-    msg "Reproducing all pkgnames listed in ./PKGBUILD"
-    set -- "${pkgnames[@]}"
-fi
-
-# check each package to see if it's a file, and if not, try to download it
-# using pacman -Sw, and get the filename from there
-splitpkgs=()
-for p in "$@"; do
-    if [[ -f ${p} ]]; then
-        splitpkgs+=("${p}")
-    else
-        pkgfile_remote=$(pacman -Sddp "${p}" 2>/dev/null) || { error "package name '%s' not in repos" "${p}"; exit 1; }
-        pkgfile=${pkgfile_remote#file://}
-        if [[ ! -f ${pkgfile} ]]; then
-            msg "Downloading package '%s' into pacman's cache" "${pkgfile}"
-            sudo pacman -Swdd --noconfirm --logfile /dev/null "${p}" || exit 1
-            pkgfile_remote=$(pacman -Sddp "${p}" 2>/dev/null)
-            pkgfile="${pkgfile_remote#file://}"
+if [[ -n $1 ]]; then
+    pkgfile="$1"
+    splitpkgs=("$@")
+    for f in "${splitpkgs[@]}"; do
+        if ! bsdtar -tqf "${f}" .BUILDINFO >/dev/null 2>&1; then
+            error "file is not a valid pacman package: '%s'" "${f}"
+            exit 1
         fi
-        splitpkgs+=("${pkgfile}")
-    fi
-done
-
-for f in "${splitpkgs[@]}"; do
-    if ! bsdtar -tqf "${f}" .BUILDINFO >/dev/null 2>&1; then
-        error "file is not a valid pacman package: '%s'" "${f}"
-        exit 1
-    fi
-done
+    done
+else
+    error "no package file specified. Try '${BASH_SOURCE[0]##*/} -h' for more information. "
+    exit 1
+fi
 
 if (( ${#cache_dirs[@]} == 0 )); then
 	mapfile -t cache_dirs < <(pacman-conf CacheDir)
@@ -174,11 +139,11 @@ load_makepkg_config
 HOME=${ORIG_HOME}
 [[ -d ${SRCDEST} ]] || SRCDEST=${PWD}
 
-parse_buildinfo < <(bsdtar -xOqf "${splitpkgs[0]}" .BUILDINFO)
+parse_buildinfo < <(bsdtar -xOqf "${pkgfile}" .BUILDINFO)
 export SOURCE_DATE_EPOCH="${buildinfo[builddate]}"
 PACKAGER="${buildinfo[packager]}"
 BUILDDIR="${buildinfo[builddir]}"
-PKGEXT=${splitpkgs[0]#${splitpkgs[0]%.pkg.tar*}}
+PKGEXT=${pkgfile#${pkgfile%.pkg.tar*}}
 
 # nuke and restore reproducible testenv
 for copy in "${buildroot}"/*/; do

offload-build.in

@@ -24,7 +24,7 @@ source /usr/share/makepkg/util/config.sh
 # global defaults suitable for use by Arch staff
 repo=extra
 arch=x86_64
-server=build.archlinux.org
+server=dragon.archlinux.org
 
 die() { printf "error: $1\n" "${@:2}"; exit 1; }
 
@@ -83,9 +83,6 @@ archbuild_cmd=("${repo}${archbuild_arch:+-$archbuild_arch}-build" "$@")
 
 trap 'rm -rf $TEMPDIR' EXIT INT TERM QUIT
 
-# Load makepkg.conf variables to be available
-load_makepkg_config
-
 # Use a source-only tarball as an intermediate to transfer files. This
 # guarantees the checksums are okay, and guarantees that all needed files are
 # transferred, including local sources, install scripts, and changelogs.
@@ -101,7 +98,7 @@ mapfile -t files < <(
     # This is sort of bash golfing but it allows running a mildly complex
     # command over ssh with a single connection.
     # shellcheck disable=SC2145
-    cat "$SRCPKGDEST"/*"$SRCEXT" |
+    cat "$SRCPKGDEST"/*.src.tar.gz |
         ssh $server '
             temp="${XDG_CACHE_HOME:-$HOME/.cache}/offload-build" &&
             mkdir -p "$temp" &&
@@ -125,6 +122,7 @@ mapfile -t files < <(
 
 if (( ${#files[@]} )); then
     printf '%s\n' '' '-> copying files...'
+    load_makepkg_config
     scp "${files[@]/#/$server:}" "${TEMPDIR}/"
     mv "${TEMPDIR}"/*.pkg.tar* "${PKGDEST:-${PWD}}/"
     mv "${TEMPDIR}/PKGBUILD" "${PWD}/"