commitpkg: introduce reuse lint and download

Signed-off-by: Morten Linderud <foxboron@archlinux.org>

Makefile

@@ -1,6 +1,6 @@
 SHELL=/bin/bash -o pipefail
 
-V=1.3.1
+V=1.3.2
 BUILDTOOLVER ?= $(V)
 
 PREFIX = /usr/local
@@ -106,7 +106,7 @@ $(BUILDDIR)/doc/man/%: doc/man/%.asciidoc doc/man/include/footer.asciidoc
 
 conf:
 	@install -d $(BUILDDIR)/makepkg.conf.d
-	@cp -a $(MAKEPKG_CONFIGS) $(BUILDDIR)/makepkg.conf.d
+	@cp -ra $(MAKEPKG_CONFIGS) $(BUILDDIR)/makepkg.conf.d
 	@install -d $(BUILDDIR)/pacman.conf.d
 	@cp -a $(PACMAN_CONFIGS) $(BUILDDIR)/pacman.conf.d
 	@install -d $(BUILDDIR)/git.conf.d
@@ -124,7 +124,7 @@ install: all
 	install -dm0755 $(DESTDIR)$(DATADIR)/lib
 	cp -ra $(BUILDDIR)/lib/* $(DESTDIR)$(DATADIR)/lib
 	cp -a $(BUILDDIR)/git.conf.d -t $(DESTDIR)$(DATADIR)
-	for conf in $(notdir $(MAKEPKG_CONFIGS)); do install -Dm0644 $(BUILDDIR)/makepkg.conf.d/$$conf $(DESTDIR)$(DATADIR)/makepkg.conf.d/$${conf##*/}; done
+	cp -ra $(BUILDDIR)/makepkg.conf.d -t $(DESTDIR)$(DATADIR)
 	for conf in $(notdir $(PACMAN_CONFIGS)); do install -Dm0644 $(BUILDDIR)/pacman.conf.d/$$conf $(DESTDIR)$(DATADIR)/pacman.conf.d/$${conf##*/}; done
 	for a in ${SETARCH_ALIASES}; do install -m0644 $$a -t $(DESTDIR)$(DATADIR)/setarch-aliases.d; done
 	for l in ${COMMITPKG_LINKS}; do ln -sf commitpkg $(DESTDIR)$(PREFIX)/bin/$$l; done
@@ -142,7 +142,7 @@ uninstall:
 	for f in $(notdir $(LIBRARY)); do rm -f $(DESTDIR)$(DATADIR)/lib/$$f; done
 	rm -rf $(DESTDIR)$(DATADIR)/lib
 	rm -rf $(DESTDIR)$(DATADIR)/git.conf.d
-	for conf in $(notdir $(MAKEPKG_CONFIGS)); do rm -f $(DESTDIR)$(DATADIR)/makepkg.conf.d/$${conf##*/}; done
+	rm -rf $(DESTDIR)$(DATADIR)/makepkg.conf.d
 	for conf in $(notdir $(PACMAN_CONFIGS)); do rm -f $(DESTDIR)$(DATADIR)/pacman.conf.d/$${conf##*/}; done
 	for f in $(notdir $(SETARCH_ALIASES)); do rm -f $(DESTDIR)$(DATADIR)/setarch-aliases.d/$$f; done
 	for l in ${COMMITPKG_LINKS}; do rm -f $(DESTDIR)$(PREFIX)/bin/$$l; done
@@ -154,7 +154,6 @@ uninstall:
 	for manfile in $(notdir $(MANS)); do rm -f $(DESTDIR)$(MANDIR)/man$${manfile##*.}/$${manfile}; done;
 	rmdir --ignore-fail-on-non-empty \
 		$(DESTDIR)$(DATADIR)/setarch-aliases.d \
-		$(DESTDIR)$(DATADIR)/makepkg.conf.d \
 		$(DESTDIR)$(DATADIR)/pacman.conf.d \
 		$(DESTDIR)$(DATADIR)
 

config/makepkg/conf.d/fortran.conf

@@ -0,0 +1,18 @@
+#!/hint/bash
+#
+# /etc/makepkg.conf.d/fortran.conf
+#
+
+#########################################################################
+# FORTRAN LANGUAGE SUPPORT
+#########################################################################
+
+# Flags used for the Fortran compiler, similar in spirit to CFLAGS. Read
+# linkman:gfortran[1] for more details on the available flags.
+#FFLAGS="-O2 -pipe"
+#FCFLAGS="$FFLAGS"
+
+# Additional compiler flags appended to `FFLAGS` and `FCFLAGS` for use in debugging. Usually
+# this would include: ``-g''. Read linkman:gfortran[1] for more details on the wide
+# variety of compiler flags available.
+#DEBUG_FFLAGS="-g"

config/makepkg/conf.d/rust.conf

@@ -0,0 +1,19 @@
+#!/hint/bash
+# shellcheck disable=2034
+
+#
+# /etc/makepkg.conf.d/rust.conf
+#
+
+#########################################################################
+# RUST LANGUAGE SUPPORT
+#########################################################################
+
+# Flags used for the Rust compiler, similar in spirit to CFLAGS. Read
+# linkman:rustc[1] for more details on the available flags.
+RUSTFLAGS="-C force-frame-pointers=yes"
+
+# Additional compiler flags appended to `RUSTFLAGS` for use in debugging.
+# Usually this would include: ``-C debuginfo=2''. Read linkman:rustc[1] for
+# more details on the available flags.
+DEBUG_RUSTFLAGS="-C debuginfo=2"

config/makepkg/x86_64.conf.d/fortran.conf

@@ -0,0 +1 @@
+../conf.d/fortran.conf

config/makepkg/x86_64.conf.d/rust.conf

@@ -0,0 +1 @@
+../conf.d/rust.conf

config/makepkg/x86_64_v3.conf.d/fortran.conf

@@ -0,0 +1 @@
+../conf.d/fortran.conf

config/makepkg/x86_64_v3.conf.d/rust.conf

@@ -0,0 +1 @@
+../conf.d/rust.conf

contrib/completion/bash/devtools.in

@@ -127,6 +127,22 @@ _sogrep() { __devtools_complete _sogrep; }
 complete -F _sogrep sogrep
 
 
+_offload_build_args=(
+	-r --repo
+	-a --arch
+	-s --server
+	-h --help
+)
+_offload_build_args__repo_opts() { _devtools_completions_build_repo; }
+_offload_build_args_r_opts() { _offload_build_args__repo_opts; }
+_offload_build_args__arch_opts() { _devtools_completions_binary_arch; }
+_offload_build_args_a_opts() { _offload_build_args__arch_opts; }
+_offload_build_args__server_opts() { :; }
+_offload_build_args_s_opts() { _offload_build_args__server_opts; }
+_offload_build() { __devtools_complete _offload_build; }
+complete -F _offload_build offload-build
+
+
 _pkgctl_cmds=(
 	aur
 	auth

contrib/completion/zsh/_devtools.in

@@ -1,4 +1,4 @@
-#compdef archbuild arch-nspawn archrelease commitpkg pkgctl diffpkg finddeps makechrootpkg mkarchroot extrapkg=commitpkg corepkg=commitpkg testingpkg=commitpkg stagingpkg=commitpkg communitypkg=commitpkg community-testingpkg=commitpkg community-stagingpkg=commitpkg multilibpkg=commitpkg multilib-testingpkg=commitpkg extra-x86_64-build=archbuild testing-x86_64-build=archbuild staging-x86_64-build=archbuild multilib-build=archbuild multilib-testing-build=archbuild multilib-staging-build=archbuild kde-unstable-x86_64-build=archbuild gnome-unstable-x86_64-build=archbuild checkpkg sogrep makerepropkg
+#compdef archbuild arch-nspawn archrelease commitpkg pkgctl diffpkg finddeps makechrootpkg mkarchroot extrapkg=commitpkg corepkg=commitpkg testingpkg=commitpkg stagingpkg=commitpkg communitypkg=commitpkg community-testingpkg=commitpkg community-stagingpkg=commitpkg multilibpkg=commitpkg multilib-testingpkg=commitpkg extra-x86_64-build=archbuild testing-x86_64-build=archbuild staging-x86_64-build=archbuild multilib-build=archbuild multilib-testing-build=archbuild multilib-staging-build=archbuild kde-unstable-x86_64-build=archbuild gnome-unstable-x86_64-build=archbuild checkpkg sogrep offload-build makerepropkg
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
@@ -61,7 +61,7 @@ _pkgctl_build_args=(
 	'--update-checksums[Force computation and update of the checksums (disables auto-detection)]'
 	'(-e --edit)'{-e,--edit}'[Edit the PKGBUILD before building]'
 	'(-r --release)'{-r,--release}'[Automatically commit, tag and release after building]'
-	'(-m --message=)'{-m,--message=}"[Use the given <msg> as the commit message]:message:"
+	'(-m --message)'{-m,--message}"[Use the given <msg> as the commit message]:message:"
 	'(-u --db-update)'{-u,--db-update}'[Automatically update the pacman database as last action]'
 	'(-h --help)'{-h,--help}'[Display usage]'
 	'*:git_dir:_files -/'
@@ -201,8 +201,8 @@ _pkgctl_issue_view_args=(
 )
 
 _pkgctl_release_args=(
-	'(-m --message=)'{-m,--message=}"[Use the given <msg> as the commit message]:message:"
-	'(-r --repo=)'{-r,--repo=}"[Specify a target repository for new packages]:repo:($DEVTOOLS_VALID_REPOS[*])"
+	'(-m --message)'{-m,--message}"[Use the given <msg> as the commit message]:message:"
+	'(-r --repo)'{-r,--repo}"[Specify a target repository for new packages]:repo:($DEVTOOLS_VALID_REPOS[*])"
 	'(-s --staging)'{-s,--staging}'[Release to the staging counterpart of the auto-detected repo]'
 	'(-t --testing)'{-t,--testing}'[Release to the testing counterpart of the auto-detected repo]'
 	'(-u --db-update)'{-u,--db-update}'[Automatically update the pacman database after uploading]'
@@ -369,6 +369,13 @@ _sogrep_args=(
 	'2:libname'
 )
 
+_offload_build_args=(
+	'(-r --repo)'{-r,--repo}'[Build against a specific repository]:repo:($DEVTOOLS_VALID_BUILDREPOS[*])'
+	'(-a --arch)'{-a,--arch}'[Build against a specific architecture]:arch:(${DEVTOOLS_VALID_BINARY_ARCHES[*]})'
+	'(-s --server)'{-s,--server}'[Offload to a specific Build server]:server:'
+	'(-h --help)'{-h,--help}'[Display usage]'
+)
+
 _makerepropkg_args=(
 	'-d[Run diffoscope if the package is unreproducible]'
 	'-n[Do not run the check() function in the PKGBUILD]'

doc/man/arch-nspawn.1.asciidoc

@@ -23,7 +23,8 @@ Options
 	Location of a pacman config file
 
 *-M* <file>::
-	Location of a makepkg config file
+	Location of a makepkg config file. Specific additions (e.g. build flags for
+	additional languages) can be placed in '<file>.d/*.conf'.
 
 *-c* <dir>::
 	Set pacman cache, if no directory is specified the passed pacman.conf's cachedir is used with a fallback to '/etc/pacman.conf'

doc/man/devtools.7.asciidoc

@@ -58,6 +58,9 @@ makechrootpkg(1)
 makerepropkg(1)
 	Rebuild a package to see if it is reproducible
 
+offload-build(1)
+	Build a PKGBUILD on a remote server using makechrootpkg
+
 sogrep(1)
 	Find packages using a linked to a given shared library
 

doc/man/makerepropkg.1.asciidoc

@@ -49,7 +49,8 @@ Options
 	Set the pacman cache directory.
 
 *-M* <file>::
-	Location of a makepkg config file.
+	Location of a makepkg config file. Specific additions (e.g. build flags for
+	additional languages) can be placed in '<file>.d/*.conf'.
 
 *-l* <chroot>::
 	The directory name to use as the chroot namespace

doc/man/offload-build.1.asciidoc

@@ -0,0 +1,52 @@
+offload-build(1)
+================
+
+Name
+----
+offload-build - Build a PKGBUILD on a remote server using makechrootpkg
+
+Synopsis
+--------
+offload-build [OPTIONS] -- [ARCHBUILD_OPTIONS]
+
+Description
+-----------
+
+Build a PKGBUILD on a remote server using makechrootpkg. Requires a remote user
+that can run archbuild in a non-interactive manner, e.g. must be able to
+elevate permissions using passwordless sudo.
+
+Options
+-------
+
+*-r, --repo* <reponame>::
+	Build against a specific repository. The default is `extra`, to build packages using
+	the stable repositories via extra-x86_64-build.
+
+*-a, --arch* <architecture>::
+	Build against a specific architecture. The default is `x86_64`, the only
+	architecture officially supported by Arch Linux.
+
+*-s, --server* <hostname>::
+	Offload to a specific build server. The default is build.archlinux.org
+	which is used as part of the build toolchain for the official Arch Linux
+	repos.
+
+*-h, --help*::
+	Show a help text.
+
+Passing options to archbuild
+----------------------------
+
+Options after a delimiting -- are passed on to archbuild on the remote.
+archbuild in turn supports passing arguments on to makechrootpkg, which in turn
+supports passing options to makepkg. Since each uses -- to delimit options that
+are forwarded, make sure to escape them properly:
+
+	`offload-build offload-args -- archbuild-args -- makechrootpkg-args -- makepkg-args`
+
+Example: To use a second `testing-x86_64-build` instance with another copydir:
+
+	`offload-build -r testing -- -- -l <chroot_copy>`
+
+include::include/footer.asciidoc[]

src/arch-nspawn.in

@@ -112,7 +112,13 @@ copy_hostconf () {
 	[[ -n $host_cachemirrors ]] && printf 'CacheServer = %s\n' "${host_cachemirrors[@]}" >>"$working_dir/etc/pacman.d/mirrorlist"
 
 	[[ -n $pac_conf ]] && cp "$pac_conf" "$working_dir/etc/pacman.conf"
-	[[ -n $makepkg_conf ]] && cp "$makepkg_conf" "$working_dir/etc/makepkg.conf"
+	if [[ -n $makepkg_conf ]]; then
+		cp "$makepkg_conf" "$working_dir/etc/makepkg.conf"
+		if [[ -d "${makepkg_conf}.d" ]] && is_globfile "${makepkg_conf}.d"/*.conf; then
+			mkdir --parents "$working_dir/etc/makepkg.conf.d/"
+			cp "${makepkg_conf}.d/"*.conf "$working_dir/etc/makepkg.conf.d/"
+		fi
+	fi
 
 	local file
 	for file in "${files[@]}"; do

src/archbuild.in

@@ -79,6 +79,13 @@ check_root SOURCE_DATE_EPOCH,SRCDEST,SRCPKGDEST,PKGDEST,LOGDEST,MAKEFLAGS,PACKAG
 # Pass all arguments after -- right to makepkg
 makechrootpkg_args+=("${@:$OPTIND}")
 
+# Automatically recreate the root chroot if a version mismatch is detected
+CURRENT_CHROOT_VERSION=$(cat "${chroots}/${repo}-${arch}/root/.arch-chroot")
+if [[ -f "${chroots}/${repo}-${arch}/root/.arch-chroot" ]] && [[ "$CURRENT_CHROOT_VERSION" != "$CHROOT_VERSION" ]]; then
+	warning "Recreating chroot '%s' (%s) as it is not at version %s" "${chroots}/${repo}-${arch}/root" "$CURRENT_CHROOT_VERSION" "$CHROOT_VERSION"
+	clean_first=true
+fi
+
 if ${clean_first} || [[ ! -d "${chroots}/${repo}-${arch}" ]]; then
 	msg "Creating chroot for [%s] (%s)..." "${repo}" "${arch}"
 

src/commitpkg.in

@@ -120,6 +120,14 @@ if (( ${#validpgpkeys[@]} != 0 )); then
 	git add --force -- keys/pgp/*
 fi
 
+# Allow us to disable REUSE
+_RUN_REUSE=${_RUN_REUSE:-1}
+if ((_RUN_REUSE)); then
+	reuse download
+	reuse lint
+	git add --force -- LICENSES/*
+fi
+
 # find files which should be under source control
 needsversioning=(PKGBUILD)
 for s in "${source[@]}"; do

src/lib/archroot.sh

@@ -4,7 +4,7 @@
 :
 
 # shellcheck disable=2034
-CHROOT_VERSION='v5'
+CHROOT_VERSION='v6'
 
 ##
 #  usage : check_root $keepenv

src/lib/build/build.sh

@@ -8,8 +8,6 @@ DEVTOOLS_INCLUDE_BUILD_SH=1
 _DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@}
 # shellcheck source=src/lib/common.sh
 source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh
-# shellcheck source=src/lib/build/offload.sh
-source "${_DEVTOOLS_LIBRARY_DIR}"/lib/build/offload.sh
 # shellcheck source=src/lib/db/update.sh
 source "${_DEVTOOLS_LIBRARY_DIR}"/lib/db/update.sh
 # shellcheck source=src/lib/release.sh
@@ -44,10 +42,10 @@ pkgctl_build_usage() {
 
 		Build packages inside a clean chroot
 
-		When a new pkgver is set using the appropriate PKGBUILD options the
-		checksums are automatically updated.
+		Build packages in clean chroot environment, offering various options
+		and functionalities to customize the package building process.
 
-		TODO
+		By default, chroot environments are located in /var/lib/archbuild/.
 
 		BUILD OPTIONS
 		    --arch ARCH          Specify architectures to build for (disables auto-detection)
@@ -465,7 +463,7 @@ pkgctl_build() {
 			fi
 
 			if (( OFFLOAD )); then
-				pkgctl_build_offload_client "${pkgbase}" "${pkgrepo}" "${arch}" "${BUILD_OPTIONS[@]}" -- "${MAKECHROOT_OPTIONS[@]}" -l "${WORKER}" -- "${MAKEPKG_OPTIONS[@]}"
+				offload-build --repo "${pkgrepo}" -- "${BUILD_OPTIONS[@]}" -- "${MAKECHROOT_OPTIONS[@]}" -l "${WORKER}" -- "${MAKEPKG_OPTIONS[@]}"
 			else
 				"${BUILDTOOL}" "${BUILD_OPTIONS[@]}" -- "${MAKECHROOT_OPTIONS[@]}" -l "${WORKER}" -- "${MAKEPKG_OPTIONS[@]}"
 			fi
@@ -483,9 +481,6 @@ pkgctl_build() {
 		# shellcheck disable=SC2119
 		write_srcinfo_file
 
-		version=$(get_full_version)
-		msg "Finished building %s %s" "${pkgbase}" "${version}"
-
 		# test-install (some of) the produced packages
 		if [[ ${INSTALL_TO_HOST} == auto ]] || [[ ${INSTALL_TO_HOST} == all ]]; then
 			# shellcheck disable=2119

src/lib/build/offload.sh

@@ -1,228 +0,0 @@
-#!/hint/bash
-
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-[[ -z ${DEVTOOLS_INCLUDE_BUILD_OFFLOAD_SH:-} ]] || return 0
-DEVTOOLS_INCLUDE_BUILD_OFFLOAD_SH=1
-
-_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@}
-# shellcheck source=src/lib/common.sh
-source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh
-# shellcheck source=src/lib/util/makepkg.sh
-source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/makepkg.sh
-
-source /usr/share/makepkg/util/config.sh
-source /usr/share/makepkg/util/message.sh
-
-set -eo pipefail
-
-
-PKGCTL_OFFLOAD_CACHE_HOME="${XDG_CACHE_HOME:-$HOME/.cache}/pkgctl/offload"
-
-pkgctl_build_offload_usage() {
-	local -r COMMAND=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}}
-	cat <<- _EOF_
-		Usage: ${COMMAND} [COMMAND] [OPTIONS]...
-
-		Server commands to build packages remotely by offloading the job.
-
-		For internal use only!
-_EOF_
-}
-
-pkgctl_build_offload() {
-	if (( $# < 1 )); then
-		pkgctl_build_offload_usage
-		exit 1
-	fi
-
-	while (( $# )); do
-		case $1 in
-			-h|--help)
-				pkgctl_build_offload_usage
-				exit 0
-				;;
-			create-builddir)
-				shift
-				pkgctl_build_offload_server_create_builddir "$@"
-				exit 0
-				;;
-			clean-builddir)
-				shift
-				pkgctl_build_offload_server_clean_builddir "$@"
-				exit 0
-				;;
-			build)
-				shift
-				pkgctl_build_offload_server_build "$@"
-				exit 0
-				;;
-			collect-files)
-				shift
-				pkgctl_build_offload_server_collect_files "$@"
-				exit 0
-				;;
-			collect-logs)
-				shift
-				pkgctl_build_offload_server_collect_logs "$@"
-				exit 0
-				;;
-			*)
-				die "invalid argument: %s" "$1"
-				;;
-		esac
-	done
-}
-
-pkgctl_build_offload_client() {
-	local pkgbase=$1
-	local pkgrepo=$2
-	local pkgarch=$3
-	shift 3
-	local server=build.archlinux.org
-	# shellcheck disable=SC2031
-	local working_dir=$PWD
-	local _srcpkg srcpkg files
-
-	[[ -z ${WORKDIR:-} ]] && setup_workdir
-	TEMPDIR=$(mktemp --tmpdir="${WORKDIR}" --directory "offload.${pkgbase}.${pkgrepo}-${pkgarch}XXXXXXXXXX")
-
-	# Load makepkg.conf variables to be available
-	# shellcheck disable=SC2119
-	load_makepkg_config
-
-	# Use a source-only tarball as an intermediate to transfer files. This
-	# guarantees the checksums are okay, and guarantees that all needed files are
-	# transferred, including local sources, install scripts, and changelogs.
-	export SRCPKGDEST="${TEMPDIR}"
-	if ! makepkg_source_package; then
-		die "unable to make source package"
-		return 1
-	fi
-
-	# Temporary cosmetic workaround makepkg if SRCDEST is set somewhere else
-	# but an empty src dir is created in PWD. Remove once fixed in makepkg.
-	rmdir --ignore-fail-on-non-empty src 2>/dev/null || true
-
-	local builddir
-	builddir=$(
-		ssh "${SSH_OPTS[@]}" -- "$server" pkgctl offload create-builddir "${pkgbase@Q}" "${pkgrepo@Q}" "${pkgarch@Q}"
-	)
-
-	# Transfer the srcpkg to the server
-	msg "Transferring source package to the server..."
-	_srcpkg=("$SRCPKGDEST"/*"$SRCEXT")
-	srcpkg="${_srcpkg[0]}"
-	if ! rsync "${RSYNC_OPTS[@]}" -- "$srcpkg" "$server":"${builddir}"; then
-		die "failed to rsync sources to offload server"
-		return 1
-	fi
-
-	# Execute build
-	if ssh "${SSH_OPTS[@]}" -t -- "$server" pkgctl offload build "${builddir@Q}" "${srcpkg@Q}" "${pkgrepo@Q}" "${pkgarch@Q}" "${@@Q}"; then
-		# Get an array of files that should be downloaded from the server
-		mapfile -t files < <(
-			ssh "${SSH_OPTS[@]}" -- "$server" pkgctl offload collect-files "${builddir@Q}" "${pkgrepo@Q}" "${pkgarch@Q}"
-		)
-	else
-		# Build failed, only the logs should be downloaded from the server
-		mapfile -t files < <(
-			ssh "${SSH_OPTS[@]}" -- "$server" pkgctl offload collect-logs "${builddir@Q}"
-		)
-	fi
-
-	# Check if we collected any files to download
-	if (( ${#files[@]} == 0 )); then
-		die "failed to collect files to download"
-		return 1
-	fi
-
-	msg 'Downloading files...'
-	rsync "${RSYNC_OPTS[@]}" -- "${files[@]/#/$server:}" "${TEMPDIR}/"
-
-	# Clean remote build dir
-	ssh "${SSH_OPTS[@]}" -- "$server" pkgctl offload clean-builddir "${builddir@Q}"
-
-	# Move all log files to LOGDEST
-	if is_globfile "${TEMPDIR}"/*.log; then
-		mv "${TEMPDIR}"/*.log "${LOGDEST:-${working_dir}}/"
-	fi
-
-	# Assume build failed if we didn't download any package files
-	if ! is_globfile "${TEMPDIR}"/*.pkg.tar*; then
-		error "Build failed, check logs in ${LOGDEST:-${working_dir}}"
-		return 1
-	fi
-
-	# Building a package may change the PKGBUILD during update_pkgver
-	mv "${TEMPDIR}/PKGBUILD" "${working_dir}/"
-	mv "${TEMPDIR}"/*.pkg.tar* "${PKGDEST:-${working_dir}}/"
-	return 0
-}
-
-pkgctl_build_offload_server_build() {
-	local builddir=$1
-	local srcpkg=$2
-	local pkgrepo=$3
-	local pkgarch=$4
-	shift 4
-	local buildtool
-
-	if [[ -n $pkgarch ]]; then
-		buildtool="${pkgrepo}-${pkgarch}-build"
-	else
-		buildtool="${pkgrepo}-build"
-	fi
-
-	cd "${builddir}"
-	bsdtar --strip-components 1 -xvf "$(basename "$srcpkg")"
-	LOGDEST="" "${buildtool}" "$@"
-}
-
-pkgctl_build_offload_server_create_builddir() {
-	local pkgbase=$1
-	local pkgrepo=$2
-	local pkgarch=$3
-	mkdir --parents "${PKGCTL_OFFLOAD_CACHE_HOME}"
-	mktemp --directory --tmpdir="${PKGCTL_OFFLOAD_CACHE_HOME}" "${pkgbase}.${pkgrepo}-${pkgarch}XXXXXXXXXX"
-}
-
-pkgctl_build_offload_server_clean_builddir() {
-	local builddir=$1
-	rm --recursive --force -- "${builddir}"
-}
-
-pkgctl_build_offload_server_collect_files() {
-	local builddir=$1
-	local pkgrepo=$2
-	local pkgarch=$3
-
-	local makepkg_config
-	local makepkg_user_config
-
-	# fallback config for multilib
-	if [[ ${pkgrepo} == multilib* ]] && [[ -z ${pkgarch} ]]; then
-		pkgarch=x86_64
-	fi
-
-	cd "${builddir}"
-	makepkg_user_config="${XDG_CONFIG_HOME:-$HOME/.config}/pacman/makepkg.conf"
-	makepkg_config="${_DEVTOOLS_LIBRARY_DIR}/makepkg.conf.d/${pkgarch}.conf"
-	if [[ -f ${_DEVTOOLS_LIBRARY_DIR}/makepkg.conf.d/${pkgrepo}-${pkgarch}.conf ]]; then
-		makepkg_config="${_DEVTOOLS_LIBRARY_DIR}/makepkg.conf.d/${pkgrepo}-${pkgarch}.conf"
-	fi
-	while read -r file; do
-		if [[ -f "${file}" ]]; then
-			printf "%s\n" "${file}"
-		fi
-	done < <(makepkg --config <(cat "${makepkg_user_config}" "${makepkg_config}" 2>/dev/null) --packagelist)
-
-	printf "%s\n" "${builddir}/PKGBUILD"
-
-	pkgctl_build_offload_server_collect_logs "${builddir}"
-}
-
-pkgctl_build_offload_server_collect_logs() {
-	local builddir=$1
-	find "${builddir}" -name "*.log"
-}

src/lib/common.sh

@@ -18,6 +18,9 @@ export LANG=C.UTF-8
 # Avoid systemd trying to color the terminal on systemd-nspawn
 export SYSTEMD_TINT_BACKGROUND=no
 
+# Avoid diffoscope looking at remote debug info through readelf
+unset DEBUGINFOD_URLS
+
 # Set buildtool properties
 export BUILDTOOL=devtools
 export BUILDTOOLVER=@buildtoolver@

src/makerepropkg.in

@@ -93,14 +93,36 @@ get_makepkg_conf() {
     local fname=${1}
     local arch="${2}"
     local makepkg_conf="${3}"
+
     if ! buildtool_file=$(get_pkgfile "${fname}"); then
-        error "failed to retrieve ${fname}"
+       error "failed to retrieve ${fname}"
+       return 1
+    fi
+    buildtool_file="${buildtool_file/file:\/\//}"
+    msg "using makepkg.conf from ${fname}"
+
+    # try to handle config of legacy devtools
+    if bsdtar --list --file "${buildtool_file}" "usr/share/devtools/makepkg-${arch}.conf" &>/dev/null; then
+        bsdtar --extract --to-stdout --fast-read --file "${buildtool_file}" "usr/share/devtools/makepkg-${arch}.conf" > "${makepkg_conf}"
+        return $?
+    fi
+
+    msg2 "extracting ${arch}.conf from devtools archive"
+    if ! bsdtar --extract --to-stdout --fast-read --file "${buildtool_file}" "usr/share/devtools/makepkg.conf.d/${arch}.conf" > "${makepkg_conf}"; then
+        error "failed to extract 'usr/share/devtools/makepkg.conf.d/${arch}.conf' from devtools archive"
         return 1
     fi
-    msg2 "using makepkg.conf from ${fname}"
-	if ! bsdtar xOqf "${buildtool_file/file:\/\//}" "usr/share/devtools/makepkg.conf.d/${arch}.conf" > "${makepkg_conf}"; then
-		bsdtar xOqf "${buildtool_file/file:\/\//}" "usr/share/devtools/makepkg-${arch}.conf" > "${makepkg_conf}"
-	fi
+
+    mkdir --parents "${makepkg_conf}.d"
+    if bsdtar --list --file "${buildtool_file}" "usr/share/devtools/makepkg.conf.d/conf.d" &>/dev/null; then
+        msg2 "extracting conf.d from devtools archive"
+        bsdtar --extract --file "${buildtool_file}" --cd "${makepkg_conf}.d" --strip-components 4 "usr/share/devtools/makepkg.conf.d/conf.d"
+    fi
+    if bsdtar --list --file "${buildtool_file}" "usr/share/devtools/makepkg.conf.d/${arch}.conf.d" &>/dev/null; then
+        msg2 "extracting ${arch}.conf.d from devtools archive"
+        bsdtar --extract --file "${buildtool_file}" --cd "${makepkg_conf}.d" --strip-components 4 "usr/share/devtools/makepkg.conf.d/${arch}.conf.d"
+    fi
+
     return 0
 }
 
@@ -186,7 +208,7 @@ for f in "${splitpkgs[@]}"; do
 done
 
 if (( ${#cache_dirs[@]} == 0 )); then
-	mapfile -t cache_dirs < <(pacman-conf CacheDir)
+    mapfile -t cache_dirs < <(pacman-conf CacheDir)
 fi
 
 ORIG_HOME=${HOME}

src/offload-build.in

@@ -0,0 +1,162 @@
+#!/bin/bash
+#
+# offload-build - build a PKGBUILD on a remote server using makechrootpkg.
+#
+# Copyright (c) 2019 by Eli Schwartz <eschwartz@archlinux.org>
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@}
+# shellcheck source=src/lib/common.sh
+source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh
+# shellcheck source=src/lib/util/makepkg.sh
+source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/makepkg.sh
+
+source /usr/share/makepkg/util/config.sh
+
+# Deprecation warning
+if [[ -z $_DEVTOOLS_COMMAND ]]; then
+	warning "${0##*/} is deprecated and will be removed. Use 'pkgctl build --offload' instead"
+fi
+
+# global defaults suitable for use by Arch staff
+repo=extra
+arch=x86_64
+server=build.archlinux.org
+
+usage() {
+    cat <<- _EOF_
+		Usage: ${BASH_SOURCE[0]##*/} [--repo REPO] [--arch ARCHITECTURE] [--server SERVER] -- [ARCHBUILD_ARGS]
+
+		Build a PKGBUILD on a remote server using makechrootpkg. Requires a remote user
+		that can run archbuild without password auth. Options passed after a -- are
+		passed on to archbuild, and eventually to makechrootpkg.
+
+		OPTIONS
+		    -r, --repo      Build against a specific repository (current: $repo)
+		    -a, --arch      Build against a specific architecture (current: $arch)
+		    -s, --server    Offload to a specific build server (current: $server)
+		    -h, --help      Show this help text
+_EOF_
+}
+
+# option checking
+while (( $# )); do
+    case $1 in
+        -h|--help)
+            usage
+            exit 0
+            ;;
+        -r|--repo)
+            repo=$2
+            shift 2
+            ;;
+        -a|--arch)
+            arch=$2
+            shift 2
+            ;;
+        -s|--server)
+            server=$2
+            shift 2
+            ;;
+        --)
+            shift
+            break
+            ;;
+        *)
+            die "invalid argument: %s" "$1"
+            ;;
+    esac
+done
+
+# multilib must be handled specially
+archbuild_arch="${arch}"
+if [[ $repo = multilib* ]]; then
+    archbuild_arch=
+fi
+
+archbuild_cmd=("${repo}${archbuild_arch:+-$archbuild_arch}-build" "$@")
+
+[[ -z ${WORKDIR:-} ]] && setup_workdir
+export TEMPDIR=$(mktemp --tmpdir="${WORKDIR}" --directory offload-build.XXXXXXXXXX)
+
+# Load makepkg.conf variables to be available
+load_makepkg_config
+
+# Use a source-only tarball as an intermediate to transfer files. This
+# guarantees the checksums are okay, and guarantees that all needed files are
+# transferred, including local sources, install scripts, and changelogs.
+export SRCPKGDEST="${TEMPDIR}"
+makepkg_source_package || die "unable to make source package"
+
+# Temporary cosmetic workaround makepkg if SRCDEST is set somewhere else
+# but an empty src dir is created in PWD. Remove once fixed in makepkg.
+rmdir --ignore-fail-on-non-empty src 2>/dev/null || true
+
+# Create a temporary directory on the server
+remote_temp=$(
+    ssh "${SSH_OPTS[@]}" -- "$server" '
+        temp="${XDG_CACHE_HOME:-$HOME/.cache}/offload-build" &&
+        mkdir -p "$temp" &&
+        mktemp --directory --tmpdir="$temp"
+')
+
+# Transfer the srcpkg to the server
+msg "Transferring source package to the server..."
+_srcpkg=("$SRCPKGDEST"/*"$SRCEXT")
+srcpkg="${_srcpkg[0]}"
+rsync "${RSYNC_OPTS[@]}" -- "$srcpkg" "$server":"$remote_temp" || die
+
+# Prepare the srcpkg on the server
+msg "Extracting srcpkg"
+ssh "${SSH_OPTS[@]}" -- "$server" "cd ${remote_temp@Q} && bsdtar --strip-components 1 -xvf $(basename "$srcpkg")" || die
+
+# Run the build command on the server
+msg "Running archbuild"
+# shellcheck disable=SC2145
+if ssh "${SSH_OPTS[@]}" -t -- "$server" "cd ${remote_temp@Q} && export LOGDEST="" && ${archbuild_cmd[@]@Q}"; then
+    msg "Build complete"
+
+    # Get an array of files that should be downloaded from the server
+    mapfile -t files < <(
+        ssh "${SSH_OPTS[@]}" -- "$server" "
+            cd ${remote_temp@Q}"' &&
+            makepkg_user_config="${XDG_CONFIG_HOME:-$HOME/.config}/pacman/makepkg.conf" &&
+            makepkg_config="/usr/share/devtools/makepkg.conf.d/'"${arch}"'.conf" &&
+            if [[ -f /usr/share/devtools/makepkg.conf.d/'"${repo}"'-'"${arch}"'.conf ]]; then
+                makepkg_config="/usr/share/devtools/makepkg.conf.d/'"${repo}"'-'"${arch}"'.conf"
+            fi &&
+            while read -r file; do
+                [[ -f "${file}" ]] && printf "%s\n" "${file}" ||:
+            done < <(makepkg --config <(cat "${makepkg_user_config}" "${makepkg_config}" 2>/dev/null) --packagelist) &&
+            printf "%s\n" '"${remote_temp@Q}/PKGBUILD"'
+
+            find '"${remote_temp@Q}"' -name "*.log"
+    ')
+else
+    # Build failed, only the logs should be downloaded from the server
+    mapfile -t files < <(
+        ssh "${SSH_OPTS[@]}" -- "$server" '
+            find '"${remote_temp@Q}"' -name "*.log"
+    ')
+fi
+
+
+if (( ${#files[@]} )); then
+    msg 'Downloading files...'
+    rsync "${RSYNC_OPTS[@]}" -- "${files[@]/#/$server:}" "${TEMPDIR}/" || die
+
+    if is_globfile "${TEMPDIR}"/*.log; then
+        mv "${TEMPDIR}"/*.log "${LOGDEST:-${PWD}}/"
+    fi
+    if is_globfile "${TEMPDIR}"/*.pkg.tar*; then
+        # Building a package may change the PKGBUILD during update_pkgver
+        mv "${TEMPDIR}/PKGBUILD" "${PWD}/"
+        mv "${TEMPDIR}"/*.pkg.tar* "${PKGDEST:-${PWD}}/"
+    else
+        error "Build failed, check logs in ${LOGDEST:-${PWD}}"
+        exit 1
+    fi
+else
+    exit 1
+fi

src/pkgctl.in

@@ -113,14 +113,6 @@ while (( $# )); do
 			pkgctl_issue "$@"
 			exit 0
 			;;
-		offload)
-			_DEVTOOLS_COMMAND+=" $1"
-			shift
-			# shellcheck source=src/lib/build/offload.sh
-			source "${_DEVTOOLS_LIBRARY_DIR}"/lib/build/offload.sh
-			pkgctl_build_offload "$@"
-			exit 0
-			;;
 		release)
 			_DEVTOOLS_COMMAND+=" $1"
 			shift