feat(build): Invoke pkgctl offload server subcommand over ssh

Use a client-server script invocation instead of piping complex shell
commands as pure strings to the remote. Now we call a server use only
component on the remote server to execute the desired build.

This change allows for properly developing the remote command without
golfind around shell strings and while having appropriate completion,
syntax highlight and LSP support. As a side-effect we are now not
depending on a POSIX compliant login shell anymore, as we simply invike
a pkgctl subcommand schema.

Component: pkgctl build
Signed-off-by: Levente Polyak <anthraxx@archlinux.org>

contrib/completion/bash/devtools.in

@@ -127,22 +127,6 @@ _sogrep() { __devtools_complete _sogrep; }
 complete -F _sogrep sogrep
 
 
-_offload_build_args=(
-	-r --repo
-	-a --arch
-	-s --server
-	-h --help
-)
-_offload_build_args__repo_opts() { _devtools_completions_build_repo; }
-_offload_build_args_r_opts() { _offload_build_args__repo_opts; }
-_offload_build_args__arch_opts() { _devtools_completions_binary_arch; }
-_offload_build_args_a_opts() { _offload_build_args__arch_opts; }
-_offload_build_args__server_opts() { :; }
-_offload_build_args_s_opts() { _offload_build_args__server_opts; }
-_offload_build() { __devtools_complete _offload_build; }
-complete -F _offload_build offload-build
-
-
 _pkgctl_cmds=(
 	aur
 	auth

contrib/completion/zsh/_devtools.in

@@ -1,4 +1,4 @@
-#compdef archbuild arch-nspawn archrelease commitpkg pkgctl diffpkg finddeps makechrootpkg mkarchroot extrapkg=commitpkg corepkg=commitpkg testingpkg=commitpkg stagingpkg=commitpkg communitypkg=commitpkg community-testingpkg=commitpkg community-stagingpkg=commitpkg multilibpkg=commitpkg multilib-testingpkg=commitpkg extra-x86_64-build=archbuild testing-x86_64-build=archbuild staging-x86_64-build=archbuild multilib-build=archbuild multilib-testing-build=archbuild multilib-staging-build=archbuild kde-unstable-x86_64-build=archbuild gnome-unstable-x86_64-build=archbuild checkpkg sogrep offload-build makerepropkg
+#compdef archbuild arch-nspawn archrelease commitpkg pkgctl diffpkg finddeps makechrootpkg mkarchroot extrapkg=commitpkg corepkg=commitpkg testingpkg=commitpkg stagingpkg=commitpkg communitypkg=commitpkg community-testingpkg=commitpkg community-stagingpkg=commitpkg multilibpkg=commitpkg multilib-testingpkg=commitpkg extra-x86_64-build=archbuild testing-x86_64-build=archbuild staging-x86_64-build=archbuild multilib-build=archbuild multilib-testing-build=archbuild multilib-staging-build=archbuild kde-unstable-x86_64-build=archbuild gnome-unstable-x86_64-build=archbuild checkpkg sogrep makerepropkg
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
@@ -369,13 +369,6 @@ _sogrep_args=(
 	'2:libname'
 )
 
-_offload_build_args=(
-	'(-r --repo)'{-r,--repo}'[Build against a specific repository]:repo:($DEVTOOLS_VALID_BUILDREPOS[*])'
-	'(-a --arch)'{-a,--arch}'[Build against a specific architecture]:arch:(${DEVTOOLS_VALID_BINARY_ARCHES[*]})'
-	'(-s --server)'{-s,--server}'[Offload to a specific Build server]:server:'
-	'(-h --help)'{-h,--help}'[Display usage]'
-)
-
 _makerepropkg_args=(
 	'-d[Run diffoscope if the package is unreproducible]'
 	'-n[Do not run the check() function in the PKGBUILD]'

doc/man/devtools.7.asciidoc

@@ -58,9 +58,6 @@ makechrootpkg(1)
 makerepropkg(1)
 	Rebuild a package to see if it is reproducible
 
-offload-build(1)
-	Build a PKGBUILD on a remote server using makechrootpkg
-
 sogrep(1)
 	Find packages using a linked to a given shared library
 

doc/man/offload-build.1.asciidoc

@@ -1,52 +0,0 @@
-offload-build(1)
-================
-
-Name
-----
-offload-build - Build a PKGBUILD on a remote server using makechrootpkg
-
-Synopsis
---------
-offload-build [OPTIONS] -- [ARCHBUILD_OPTIONS]
-
-Description
------------
-
-Build a PKGBUILD on a remote server using makechrootpkg. Requires a remote user
-that can run archbuild in a non-interactive manner, e.g. must be able to
-elevate permissions using passwordless sudo.
-
-Options
--------
-
-*-r, --repo* <reponame>::
-	Build against a specific repository. The default is `extra`, to build packages using
-	the stable repositories via extra-x86_64-build.
-
-*-a, --arch* <architecture>::
-	Build against a specific architecture. The default is `x86_64`, the only
-	architecture officially supported by Arch Linux.
-
-*-s, --server* <hostname>::
-	Offload to a specific build server. The default is build.archlinux.org
-	which is used as part of the build toolchain for the official Arch Linux
-	repos.
-
-*-h, --help*::
-	Show a help text.
-
-Passing options to archbuild
-----------------------------
-
-Options after a delimiting -- are passed on to archbuild on the remote.
-archbuild in turn supports passing arguments on to makechrootpkg, which in turn
-supports passing options to makepkg. Since each uses -- to delimit options that
-are forwarded, make sure to escape them properly:
-
-	`offload-build offload-args -- archbuild-args -- makechrootpkg-args -- makepkg-args`
-
-Example: To use a second `testing-x86_64-build` instance with another copydir:
-
-	`offload-build -r testing -- -- -l <chroot_copy>`
-
-include::include/footer.asciidoc[]

src/lib/build/build.sh

@@ -8,6 +8,8 @@ DEVTOOLS_INCLUDE_BUILD_SH=1
 _DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@}
 # shellcheck source=src/lib/common.sh
 source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh
+# shellcheck source=src/lib/build/offload.sh
+source "${_DEVTOOLS_LIBRARY_DIR}"/lib/build/offload.sh
 # shellcheck source=src/lib/db/update.sh
 source "${_DEVTOOLS_LIBRARY_DIR}"/lib/db/update.sh
 # shellcheck source=src/lib/release.sh
@@ -463,7 +465,7 @@ pkgctl_build() {
 			fi
 
 			if (( OFFLOAD )); then
-				offload-build --repo "${pkgrepo}" -- "${BUILD_OPTIONS[@]}" -- "${MAKECHROOT_OPTIONS[@]}" -l "${WORKER}" -- "${MAKEPKG_OPTIONS[@]}"
+				pkgctl_build_offload_client "${pkgbase}" "${pkgrepo}" "${arch}" "${BUILD_OPTIONS[@]}" -- "${MAKECHROOT_OPTIONS[@]}" -l "${WORKER}" -- "${MAKEPKG_OPTIONS[@]}"
 			else
 				"${BUILDTOOL}" "${BUILD_OPTIONS[@]}" -- "${MAKECHROOT_OPTIONS[@]}" -l "${WORKER}" -- "${MAKEPKG_OPTIONS[@]}"
 			fi
@@ -481,6 +483,9 @@ pkgctl_build() {
 		# shellcheck disable=SC2119
 		write_srcinfo_file
 
+		version=$(get_full_version)
+		msg "Finished building %s %s" "${pkgbase}" "${version}"
+
 		# test-install (some of) the produced packages
 		if [[ ${INSTALL_TO_HOST} == auto ]] || [[ ${INSTALL_TO_HOST} == all ]]; then
 			# shellcheck disable=2119

src/lib/build/offload.sh

@@ -0,0 +1,228 @@
+#!/hint/bash
+
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+[[ -z ${DEVTOOLS_INCLUDE_BUILD_OFFLOAD_SH:-} ]] || return 0
+DEVTOOLS_INCLUDE_BUILD_OFFLOAD_SH=1
+
+_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@}
+# shellcheck source=src/lib/common.sh
+source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh
+# shellcheck source=src/lib/util/makepkg.sh
+source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/makepkg.sh
+
+source /usr/share/makepkg/util/config.sh
+source /usr/share/makepkg/util/message.sh
+
+set -eo pipefail
+
+
+PKGCTL_OFFLOAD_CACHE_HOME="${XDG_CACHE_HOME:-$HOME/.cache}/pkgctl/offload"
+
+pkgctl_build_offload_usage() {
+	local -r COMMAND=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}}
+	cat <<- _EOF_
+		Usage: ${COMMAND} [COMMAND] [OPTIONS]...
+
+		Server commands to build packages remotely by offloading the job.
+
+		For internal use only!
+_EOF_
+}
+
+pkgctl_build_offload() {
+	if (( $# < 1 )); then
+		pkgctl_build_offload_usage
+		exit 1
+	fi
+
+	while (( $# )); do
+		case $1 in
+			-h|--help)
+				pkgctl_build_offload_usage
+				exit 0
+				;;
+			create-builddir)
+				shift
+				pkgctl_build_offload_server_create_builddir "$@"
+				exit 0
+				;;
+			clean-builddir)
+				shift
+				pkgctl_build_offload_server_clean_builddir "$@"
+				exit 0
+				;;
+			build)
+				shift
+				pkgctl_build_offload_server_build "$@"
+				exit 0
+				;;
+			collect-files)
+				shift
+				pkgctl_build_offload_server_collect_files "$@"
+				exit 0
+				;;
+			collect-logs)
+				shift
+				pkgctl_build_offload_server_collect_logs "$@"
+				exit 0
+				;;
+			*)
+				die "invalid argument: %s" "$1"
+				;;
+		esac
+	done
+}
+
+pkgctl_build_offload_client() {
+	local pkgbase=$1
+	local pkgrepo=$2
+	local pkgarch=$3
+	shift 3
+	local server=build.archlinux.org
+	# shellcheck disable=SC2031
+	local working_dir=$PWD
+	local _srcpkg srcpkg files
+
+	[[ -z ${WORKDIR:-} ]] && setup_workdir
+	TEMPDIR=$(mktemp --tmpdir="${WORKDIR}" --directory "offload.${pkgbase}.${pkgrepo}-${pkgarch}XXXXXXXXXX")
+
+	# Load makepkg.conf variables to be available
+	# shellcheck disable=SC2119
+	load_makepkg_config
+
+	# Use a source-only tarball as an intermediate to transfer files. This
+	# guarantees the checksums are okay, and guarantees that all needed files are
+	# transferred, including local sources, install scripts, and changelogs.
+	export SRCPKGDEST="${TEMPDIR}"
+	if ! makepkg_source_package; then
+		die "unable to make source package"
+		return 1
+	fi
+
+	# Temporary cosmetic workaround makepkg if SRCDEST is set somewhere else
+	# but an empty src dir is created in PWD. Remove once fixed in makepkg.
+	rmdir --ignore-fail-on-non-empty src 2>/dev/null || true
+
+	local builddir
+	builddir=$(
+		ssh "${SSH_OPTS[@]}" -- "$server" pkgctl offload create-builddir "${pkgbase@Q}" "${pkgrepo@Q}" "${pkgarch@Q}"
+	)
+
+	# Transfer the srcpkg to the server
+	msg "Transferring source package to the server..."
+	_srcpkg=("$SRCPKGDEST"/*"$SRCEXT")
+	srcpkg="${_srcpkg[0]}"
+	if ! rsync "${RSYNC_OPTS[@]}" -- "$srcpkg" "$server":"${builddir}"; then
+		die "failed to rsync sources to offload server"
+		return 1
+	fi
+
+	# Execute build
+	if ssh "${SSH_OPTS[@]}" -t -- "$server" pkgctl offload build "${builddir@Q}" "${srcpkg@Q}" "${pkgrepo@Q}" "${pkgarch@Q}" "${@@Q}"; then
+		# Get an array of files that should be downloaded from the server
+		mapfile -t files < <(
+			ssh "${SSH_OPTS[@]}" -- "$server" pkgctl offload collect-files "${builddir@Q}" "${pkgrepo@Q}" "${pkgarch@Q}"
+		)
+	else
+		# Build failed, only the logs should be downloaded from the server
+		mapfile -t files < <(
+			ssh "${SSH_OPTS[@]}" -- "$server" pkgctl offload collect-logs "${builddir@Q}"
+		)
+	fi
+
+	# Check if we collected any files to download
+	if (( ${#files[@]} == 0 )); then
+		die "failed to collect files to download"
+		return 1
+	fi
+
+	msg 'Downloading files...'
+	rsync "${RSYNC_OPTS[@]}" -- "${files[@]/#/$server:}" "${TEMPDIR}/"
+
+	# Clean remote build dir
+	ssh "${SSH_OPTS[@]}" -- "$server" pkgctl offload clean-builddir "${builddir@Q}"
+
+	# Move all log files to LOGDEST
+	if is_globfile "${TEMPDIR}"/*.log; then
+		mv "${TEMPDIR}"/*.log "${LOGDEST:-${working_dir}}/"
+	fi
+
+	# Assume build failed if we didn't download any package files
+	if ! is_globfile "${TEMPDIR}"/*.pkg.tar*; then
+		error "Build failed, check logs in ${LOGDEST:-${working_dir}}"
+		return 1
+	fi
+
+	# Building a package may change the PKGBUILD during update_pkgver
+	mv "${TEMPDIR}/PKGBUILD" "${working_dir}/"
+	mv "${TEMPDIR}"/*.pkg.tar* "${PKGDEST:-${working_dir}}/"
+	return 0
+}
+
+pkgctl_build_offload_server_build() {
+	local builddir=$1
+	local srcpkg=$2
+	local pkgrepo=$3
+	local pkgarch=$4
+	shift 4
+	local buildtool
+
+	if [[ -n $pkgarch ]]; then
+		buildtool="${pkgrepo}-${pkgarch}-build"
+	else
+		buildtool="${pkgrepo}-build"
+	fi
+
+	cd "${builddir}"
+	bsdtar --strip-components 1 -xvf "$(basename "$srcpkg")"
+	LOGDEST="" "${buildtool}" "$@"
+}
+
+pkgctl_build_offload_server_create_builddir() {
+	local pkgbase=$1
+	local pkgrepo=$2
+	local pkgarch=$3
+	mkdir --parents "${PKGCTL_OFFLOAD_CACHE_HOME}"
+	mktemp --directory --tmpdir="${PKGCTL_OFFLOAD_CACHE_HOME}" "${pkgbase}.${pkgrepo}-${pkgarch}XXXXXXXXXX"
+}
+
+pkgctl_build_offload_server_clean_builddir() {
+	local builddir=$1
+	rm --recursive --force -- "${builddir}"
+}
+
+pkgctl_build_offload_server_collect_files() {
+	local builddir=$1
+	local pkgrepo=$2
+	local pkgarch=$3
+
+	local makepkg_config
+	local makepkg_user_config
+
+	# fallback config for multilib
+	if [[ ${pkgrepo} == multilib* ]] && [[ -z ${pkgarch} ]]; then
+		pkgarch=x86_64
+	fi
+
+	cd "${builddir}"
+	makepkg_user_config="${XDG_CONFIG_HOME:-$HOME/.config}/pacman/makepkg.conf"
+	makepkg_config="${_DEVTOOLS_LIBRARY_DIR}/makepkg.conf.d/${pkgarch}.conf"
+	if [[ -f ${_DEVTOOLS_LIBRARY_DIR}/makepkg.conf.d/${pkgrepo}-${pkgarch}.conf ]]; then
+		makepkg_config="${_DEVTOOLS_LIBRARY_DIR}/makepkg.conf.d/${pkgrepo}-${pkgarch}.conf"
+	fi
+	while read -r file; do
+		if [[ -f "${file}" ]]; then
+			printf "%s\n" "${file}"
+		fi
+	done < <(makepkg --config <(cat "${makepkg_user_config}" "${makepkg_config}" 2>/dev/null) --packagelist)
+
+	printf "%s\n" "${builddir}/PKGBUILD"
+
+	pkgctl_build_offload_server_collect_logs "${builddir}"
+}
+
+pkgctl_build_offload_server_collect_logs() {
+	local builddir=$1
+	find "${builddir}" -name "*.log"
+}

src/offload-build.in

@@ -1,162 +0,0 @@
-#!/bin/bash
-#
-# offload-build - build a PKGBUILD on a remote server using makechrootpkg.
-#
-# Copyright (c) 2019 by Eli Schwartz <eschwartz@archlinux.org>
-#
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@}
-# shellcheck source=src/lib/common.sh
-source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh
-# shellcheck source=src/lib/util/makepkg.sh
-source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/makepkg.sh
-
-source /usr/share/makepkg/util/config.sh
-
-# Deprecation warning
-if [[ -z $_DEVTOOLS_COMMAND ]]; then
-	warning "${0##*/} is deprecated and will be removed. Use 'pkgctl build --offload' instead"
-fi
-
-# global defaults suitable for use by Arch staff
-repo=extra
-arch=x86_64
-server=build.archlinux.org
-
-usage() {
-    cat <<- _EOF_
-		Usage: ${BASH_SOURCE[0]##*/} [--repo REPO] [--arch ARCHITECTURE] [--server SERVER] -- [ARCHBUILD_ARGS]
-
-		Build a PKGBUILD on a remote server using makechrootpkg. Requires a remote user
-		that can run archbuild without password auth. Options passed after a -- are
-		passed on to archbuild, and eventually to makechrootpkg.
-
-		OPTIONS
-		    -r, --repo      Build against a specific repository (current: $repo)
-		    -a, --arch      Build against a specific architecture (current: $arch)
-		    -s, --server    Offload to a specific build server (current: $server)
-		    -h, --help      Show this help text
-_EOF_
-}
-
-# option checking
-while (( $# )); do
-    case $1 in
-        -h|--help)
-            usage
-            exit 0
-            ;;
-        -r|--repo)
-            repo=$2
-            shift 2
-            ;;
-        -a|--arch)
-            arch=$2
-            shift 2
-            ;;
-        -s|--server)
-            server=$2
-            shift 2
-            ;;
-        --)
-            shift
-            break
-            ;;
-        *)
-            die "invalid argument: %s" "$1"
-            ;;
-    esac
-done
-
-# multilib must be handled specially
-archbuild_arch="${arch}"
-if [[ $repo = multilib* ]]; then
-    archbuild_arch=
-fi
-
-archbuild_cmd=("${repo}${archbuild_arch:+-$archbuild_arch}-build" "$@")
-
-[[ -z ${WORKDIR:-} ]] && setup_workdir
-export TEMPDIR=$(mktemp --tmpdir="${WORKDIR}" --directory offload-build.XXXXXXXXXX)
-
-# Load makepkg.conf variables to be available
-load_makepkg_config
-
-# Use a source-only tarball as an intermediate to transfer files. This
-# guarantees the checksums are okay, and guarantees that all needed files are
-# transferred, including local sources, install scripts, and changelogs.
-export SRCPKGDEST="${TEMPDIR}"
-makepkg_source_package || die "unable to make source package"
-
-# Temporary cosmetic workaround makepkg if SRCDEST is set somewhere else
-# but an empty src dir is created in PWD. Remove once fixed in makepkg.
-rmdir --ignore-fail-on-non-empty src 2>/dev/null || true
-
-# Create a temporary directory on the server
-remote_temp=$(
-    ssh "${SSH_OPTS[@]}" -- "$server" '
-        temp="${XDG_CACHE_HOME:-$HOME/.cache}/offload-build" &&
-        mkdir -p "$temp" &&
-        mktemp --directory --tmpdir="$temp"
-')
-
-# Transfer the srcpkg to the server
-msg "Transferring source package to the server..."
-_srcpkg=("$SRCPKGDEST"/*"$SRCEXT")
-srcpkg="${_srcpkg[0]}"
-rsync "${RSYNC_OPTS[@]}" -- "$srcpkg" "$server":"$remote_temp" || die
-
-# Prepare the srcpkg on the server
-msg "Extracting srcpkg"
-ssh "${SSH_OPTS[@]}" -- "$server" "cd ${remote_temp@Q} && bsdtar --strip-components 1 -xvf $(basename "$srcpkg")" || die
-
-# Run the build command on the server
-msg "Running archbuild"
-# shellcheck disable=SC2145
-if ssh "${SSH_OPTS[@]}" -t -- "$server" "cd ${remote_temp@Q} && export LOGDEST="" && ${archbuild_cmd[@]@Q}"; then
-    msg "Build complete"
-
-    # Get an array of files that should be downloaded from the server
-    mapfile -t files < <(
-        ssh "${SSH_OPTS[@]}" -- "$server" "
-            cd ${remote_temp@Q}"' &&
-            makepkg_user_config="${XDG_CONFIG_HOME:-$HOME/.config}/pacman/makepkg.conf" &&
-            makepkg_config="/usr/share/devtools/makepkg.conf.d/'"${arch}"'.conf" &&
-            if [[ -f /usr/share/devtools/makepkg.conf.d/'"${repo}"'-'"${arch}"'.conf ]]; then
-                makepkg_config="/usr/share/devtools/makepkg.conf.d/'"${repo}"'-'"${arch}"'.conf"
-            fi &&
-            while read -r file; do
-                [[ -f "${file}" ]] && printf "%s\n" "${file}" ||:
-            done < <(makepkg --config <(cat "${makepkg_user_config}" "${makepkg_config}" 2>/dev/null) --packagelist) &&
-            printf "%s\n" '"${remote_temp@Q}/PKGBUILD"'
-
-            find '"${remote_temp@Q}"' -name "*.log"
-    ')
-else
-    # Build failed, only the logs should be downloaded from the server
-    mapfile -t files < <(
-        ssh "${SSH_OPTS[@]}" -- "$server" '
-            find '"${remote_temp@Q}"' -name "*.log"
-    ')
-fi
-
-
-if (( ${#files[@]} )); then
-    msg 'Downloading files...'
-    rsync "${RSYNC_OPTS[@]}" -- "${files[@]/#/$server:}" "${TEMPDIR}/" || die
-
-    if is_globfile "${TEMPDIR}"/*.log; then
-        mv "${TEMPDIR}"/*.log "${LOGDEST:-${PWD}}/"
-    fi
-    if is_globfile "${TEMPDIR}"/*.pkg.tar*; then
-        # Building a package may change the PKGBUILD during update_pkgver
-        mv "${TEMPDIR}/PKGBUILD" "${PWD}/"
-        mv "${TEMPDIR}"/*.pkg.tar* "${PKGDEST:-${PWD}}/"
-    else
-        error "Build failed, check logs in ${LOGDEST:-${PWD}}"
-        exit 1
-    fi
-else
-    exit 1
-fi

src/pkgctl.in

@@ -113,6 +113,14 @@ while (( $# )); do
 			pkgctl_issue "$@"
 			exit 0
 			;;
+		offload)
+			_DEVTOOLS_COMMAND+=" $1"
+			shift
+			# shellcheck source=src/lib/build/offload.sh
+			source "${_DEVTOOLS_LIBRARY_DIR}"/lib/build/offload.sh
+			pkgctl_build_offload "$@"
+			exit 0
+			;;
 		release)
 			_DEVTOOLS_COMMAND+=" $1"
 			shift