feat(repo): Add the repo pull command to fetch and integrate changes

The command can be used to bring all repositories up to date with their
respective upstream repositories hosted on gitlab. On top several
subcommand options allow to enhance the output by printing the changed
diff as well as control behavior like discarding changes or auto
stashing on pull.

Fixes #211

Component: pkgctl repo pull
Co-authored-by: Levente Polyak <anthraxx@archlinux.org>
Signed-off-by: Christian Heusel <christian@heusel.eu>

Makefile

@@ -1,6 +1,6 @@
 SHELL=/bin/bash -o pipefail
 
-V=1.3.1
+V=1.3.2
 BUILDTOOLVER ?= $(V)
 
 PREFIX = /usr/local
@@ -106,7 +106,7 @@ $(BUILDDIR)/doc/man/%: doc/man/%.asciidoc doc/man/include/footer.asciidoc
 
 conf:
 	@install -d $(BUILDDIR)/makepkg.conf.d
-	@cp -a $(MAKEPKG_CONFIGS) $(BUILDDIR)/makepkg.conf.d
+	@cp -ra $(MAKEPKG_CONFIGS) $(BUILDDIR)/makepkg.conf.d
 	@install -d $(BUILDDIR)/pacman.conf.d
 	@cp -a $(PACMAN_CONFIGS) $(BUILDDIR)/pacman.conf.d
 	@install -d $(BUILDDIR)/git.conf.d
@@ -124,7 +124,7 @@ install: all
 	install -dm0755 $(DESTDIR)$(DATADIR)/lib
 	cp -ra $(BUILDDIR)/lib/* $(DESTDIR)$(DATADIR)/lib
 	cp -a $(BUILDDIR)/git.conf.d -t $(DESTDIR)$(DATADIR)
-	for conf in $(notdir $(MAKEPKG_CONFIGS)); do install -Dm0644 $(BUILDDIR)/makepkg.conf.d/$$conf $(DESTDIR)$(DATADIR)/makepkg.conf.d/$${conf##*/}; done
+	cp -ra $(BUILDDIR)/makepkg.conf.d -t $(DESTDIR)$(DATADIR)
 	for conf in $(notdir $(PACMAN_CONFIGS)); do install -Dm0644 $(BUILDDIR)/pacman.conf.d/$$conf $(DESTDIR)$(DATADIR)/pacman.conf.d/$${conf##*/}; done
 	for a in ${SETARCH_ALIASES}; do install -m0644 $$a -t $(DESTDIR)$(DATADIR)/setarch-aliases.d; done
 	for l in ${COMMITPKG_LINKS}; do ln -sf commitpkg $(DESTDIR)$(PREFIX)/bin/$$l; done
@@ -142,7 +142,7 @@ uninstall:
 	for f in $(notdir $(LIBRARY)); do rm -f $(DESTDIR)$(DATADIR)/lib/$$f; done
 	rm -rf $(DESTDIR)$(DATADIR)/lib
 	rm -rf $(DESTDIR)$(DATADIR)/git.conf.d
-	for conf in $(notdir $(MAKEPKG_CONFIGS)); do rm -f $(DESTDIR)$(DATADIR)/makepkg.conf.d/$${conf##*/}; done
+	rm -rf $(DESTDIR)$(DATADIR)/makepkg.conf.d
 	for conf in $(notdir $(PACMAN_CONFIGS)); do rm -f $(DESTDIR)$(DATADIR)/pacman.conf.d/$${conf##*/}; done
 	for f in $(notdir $(SETARCH_ALIASES)); do rm -f $(DESTDIR)$(DATADIR)/setarch-aliases.d/$$f; done
 	for l in ${COMMITPKG_LINKS}; do rm -f $(DESTDIR)$(PREFIX)/bin/$$l; done
@@ -154,7 +154,6 @@ uninstall:
 	for manfile in $(notdir $(MANS)); do rm -f $(DESTDIR)$(MANDIR)/man$${manfile##*.}/$${manfile}; done;
 	rmdir --ignore-fail-on-non-empty \
 		$(DESTDIR)$(DATADIR)/setarch-aliases.d \
-		$(DESTDIR)$(DATADIR)/makepkg.conf.d \
 		$(DESTDIR)$(DATADIR)/pacman.conf.d \
 		$(DESTDIR)$(DATADIR)
 

config/makepkg/conf.d/fortran.conf

@@ -0,0 +1,18 @@
+#!/hint/bash
+#
+# /etc/makepkg.conf.d/fortran.conf
+#
+
+#########################################################################
+# FORTRAN LANGUAGE SUPPORT
+#########################################################################
+
+# Flags used for the Fortran compiler, similar in spirit to CFLAGS. Read
+# linkman:gfortran[1] for more details on the available flags.
+#FFLAGS="-O2 -pipe"
+#FCFLAGS="$FFLAGS"
+
+# Additional compiler flags appended to `FFLAGS` and `FCFLAGS` for use in debugging. Usually
+# this would include: ``-g''. Read linkman:gfortran[1] for more details on the wide
+# variety of compiler flags available.
+#DEBUG_FFLAGS="-g"

config/makepkg/conf.d/rust.conf

@@ -0,0 +1,19 @@
+#!/hint/bash
+# shellcheck disable=2034
+
+#
+# /etc/makepkg.conf.d/rust.conf
+#
+
+#########################################################################
+# RUST LANGUAGE SUPPORT
+#########################################################################
+
+# Flags used for the Rust compiler, similar in spirit to CFLAGS. Read
+# linkman:rustc[1] for more details on the available flags.
+RUSTFLAGS="-C force-frame-pointers=yes"
+
+# Additional compiler flags appended to `RUSTFLAGS` for use in debugging.
+# Usually this would include: ``-C debuginfo=2''. Read linkman:rustc[1] for
+# more details on the available flags.
+DEBUG_RUSTFLAGS="-C debuginfo=2"

config/makepkg/x86_64.conf.d/fortran.conf

@@ -0,0 +1 @@
+../conf.d/fortran.conf

config/makepkg/x86_64.conf.d/rust.conf

@@ -0,0 +1 @@
+../conf.d/rust.conf

config/makepkg/x86_64_v3.conf.d/fortran.conf

@@ -0,0 +1 @@
+../conf.d/fortran.conf

config/makepkg/x86_64_v3.conf.d/rust.conf

@@ -0,0 +1 @@
+../conf.d/rust.conf

contrib/completion/bash/devtools.in

@@ -127,6 +127,22 @@ _sogrep() { __devtools_complete _sogrep; }
 complete -F _sogrep sogrep
 
 
+_offload_build_args=(
+	-r --repo
+	-a --arch
+	-s --server
+	-h --help
+)
+_offload_build_args__repo_opts() { _devtools_completions_build_repo; }
+_offload_build_args_r_opts() { _offload_build_args__repo_opts; }
+_offload_build_args__arch_opts() { _devtools_completions_binary_arch; }
+_offload_build_args_a_opts() { _offload_build_args__arch_opts; }
+_offload_build_args__server_opts() { :; }
+_offload_build_args_s_opts() { _offload_build_args__server_opts; }
+_offload_build() { __devtools_complete _offload_build; }
+complete -F _offload_build offload-build
+
+
 _pkgctl_cmds=(
 	aur
 	auth
@@ -289,6 +305,7 @@ _pkgctl_repo_cmds=(
 	clone
 	configure
 	create
+	pull
 	switch
 	web
 )
@@ -318,6 +335,19 @@ _pkgctl_repo_clean_args=(
 )
 _pkgctl_repo_clean_opts() { _filedir -d; }
 
+_pkgctl_repo_pull_args=(
+	--discard-changes
+	--show-diff
+	--quiet
+	--autostash
+	-f --force
+	-j --jobs
+	-h --help
+)
+_pkgctl_repo_pull_args__jobs_opts() { :; }
+_pkgctl_repo_pull_args_j_opts() { _pkgctl_repo_pull_args__jobs_opts; }
+_pkgctl_repo_pull_opts() { _filedir -d; }
+
 _pkgctl_repo_configure_args=(
 	--protocol
 	-j --jobs

contrib/completion/zsh/_devtools.in

@@ -1,4 +1,4 @@
-#compdef archbuild arch-nspawn archrelease commitpkg pkgctl diffpkg finddeps makechrootpkg mkarchroot extrapkg=commitpkg corepkg=commitpkg testingpkg=commitpkg stagingpkg=commitpkg communitypkg=commitpkg community-testingpkg=commitpkg community-stagingpkg=commitpkg multilibpkg=commitpkg multilib-testingpkg=commitpkg extra-x86_64-build=archbuild testing-x86_64-build=archbuild staging-x86_64-build=archbuild multilib-build=archbuild multilib-testing-build=archbuild multilib-staging-build=archbuild kde-unstable-x86_64-build=archbuild gnome-unstable-x86_64-build=archbuild checkpkg sogrep makerepropkg
+#compdef archbuild arch-nspawn archrelease commitpkg pkgctl diffpkg finddeps makechrootpkg mkarchroot extrapkg=commitpkg corepkg=commitpkg testingpkg=commitpkg stagingpkg=commitpkg communitypkg=commitpkg community-testingpkg=commitpkg community-stagingpkg=commitpkg multilibpkg=commitpkg multilib-testingpkg=commitpkg extra-x86_64-build=archbuild testing-x86_64-build=archbuild staging-x86_64-build=archbuild multilib-build=archbuild multilib-testing-build=archbuild multilib-staging-build=archbuild kde-unstable-x86_64-build=archbuild gnome-unstable-x86_64-build=archbuild checkpkg sogrep offload-build makerepropkg
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
@@ -202,7 +202,7 @@ _pkgctl_issue_view_args=(
 
 _pkgctl_release_args=(
 	'(-m --message=)'{-m,--message=}"[Use the given <msg> as the commit message]:message:"
-	'(-r --repo=)'{-r,--repo=}"[Specify a target repository for new packages]:repo:($DEVTOOLS_VALID_REPOS[*])"
+	'(-r --repo)'{-r,--repo}"[Specify a target repository for new packages]:repo:($DEVTOOLS_VALID_REPOS[*])"
 	'(-s --staging)'{-s,--staging}'[Release to the staging counterpart of the auto-detected repo]'
 	'(-t --testing)'{-t,--testing}'[Release to the testing counterpart of the auto-detected repo]'
 	'(-u --db-update)'{-u,--db-update}'[Automatically update the pacman database after uploading]'
@@ -228,6 +228,7 @@ _pkgctl_repo_cmds=(
 	"clone[Clone a package repository]"
 	"configure[Configure a clone according to distro specs]"
 	"create[Create a new GitLab package repository]"
+	"pull[Update package repositories from their git remote]"
 	"switch[Switch a package repository to a specified version]"
 	"web[Open the packaging repository's website]"
 )
@@ -256,6 +257,17 @@ _pkgctl_repo_clone_args=(
 	'*:packages:_devtools_completions_all_packages'
 )
 
+_pkgctl_repo_pull_args=(
+	'--discard-changes[Discard changes if index or working tree is dirty]'
+	'--quiet[Disable printing longer terminal output]'
+	'--show-diff[Always enable showing the diff]'
+	'--autostash[Stash before pulling and unstash afterwards]'
+	'(-f --force)'{-f,--force}'[Alias for --discard-changes]'
+	'(-j --jobs)'{-j,--jobs}'[Run up to N jobs in parallel (default: number of processing units)]:jobs:'
+	'(-h --help)'{-h,--help}'[Display usage]'
+	'*:git_dir:_files -/'
+)
+
 _pkgctl_repo_configure_args=(
 	'--protocol[Configure remote url to use https]:proto:(https)'
 	'(-j --jobs)'{-j,--jobs}'[Run up to N jobs in parallel (default: number of processing units)]:jobs:'
@@ -369,6 +381,13 @@ _sogrep_args=(
 	'2:libname'
 )
 
+_offload_build_args=(
+	'(-r --repo)'{-r,--repo}'[Build against a specific repository]:repo:($DEVTOOLS_VALID_BUILDREPOS[*])'
+	'(-a --arch)'{-a,--arch}'[Build against a specific architecture]:arch:(${DEVTOOLS_VALID_BINARY_ARCHES[*]})'
+	'(-s --server)'{-s,--server}'[Offload to a specific Build server]:server:'
+	'(-h --help)'{-h,--help}'[Display usage]'
+)
+
 _makerepropkg_args=(
 	'-d[Run diffoscope if the package is unreproducible]'
 	'-n[Do not run the check() function in the PKGBUILD]'

doc/man/arch-nspawn.1.asciidoc

@@ -23,7 +23,8 @@ Options
 	Location of a pacman config file
 
 *-M* <file>::
-	Location of a makepkg config file
+	Location of a makepkg config file. Specific additions (e.g. build flags for
+	additional languages) can be placed in '<file>.d/*.conf'.
 
 *-c* <dir>::
 	Set pacman cache, if no directory is specified the passed pacman.conf's cachedir is used with a fallback to '/etc/pacman.conf'

doc/man/devtools.7.asciidoc

@@ -58,6 +58,9 @@ makechrootpkg(1)
 makerepropkg(1)
 	Rebuild a package to see if it is reproducible
 
+offload-build(1)
+	Build a PKGBUILD on a remote server using makechrootpkg
+
 sogrep(1)
 	Find packages using a linked to a given shared library
 

doc/man/makerepropkg.1.asciidoc

@@ -49,7 +49,8 @@ Options
 	Set the pacman cache directory.
 
 *-M* <file>::
-	Location of a makepkg config file.
+	Location of a makepkg config file. Specific additions (e.g. build flags for
+	additional languages) can be placed in '<file>.d/*.conf'.
 
 *-l* <chroot>::
 	The directory name to use as the chroot namespace

doc/man/offload-build.1.asciidoc

@@ -0,0 +1,52 @@
+offload-build(1)
+================
+
+Name
+----
+offload-build - Build a PKGBUILD on a remote server using makechrootpkg
+
+Synopsis
+--------
+offload-build [OPTIONS] -- [ARCHBUILD_OPTIONS]
+
+Description
+-----------
+
+Build a PKGBUILD on a remote server using makechrootpkg. Requires a remote user
+that can run archbuild in a non-interactive manner, e.g. must be able to
+elevate permissions using passwordless sudo.
+
+Options
+-------
+
+*-r, --repo* <reponame>::
+	Build against a specific repository. The default is `extra`, to build packages using
+	the stable repositories via extra-x86_64-build.
+
+*-a, --arch* <architecture>::
+	Build against a specific architecture. The default is `x86_64`, the only
+	architecture officially supported by Arch Linux.
+
+*-s, --server* <hostname>::
+	Offload to a specific build server. The default is build.archlinux.org
+	which is used as part of the build toolchain for the official Arch Linux
+	repos.
+
+*-h, --help*::
+	Show a help text.
+
+Passing options to archbuild
+----------------------------
+
+Options after a delimiting -- are passed on to archbuild on the remote.
+archbuild in turn supports passing arguments on to makechrootpkg, which in turn
+supports passing options to makepkg. Since each uses -- to delimit options that
+are forwarded, make sure to escape them properly:
+
+	`offload-build offload-args -- archbuild-args -- makechrootpkg-args -- makepkg-args`
+
+Example: To use a second `testing-x86_64-build` instance with another copydir:
+
+	`offload-build -r testing -- -- -l <chroot_copy>`
+
+include::include/footer.asciidoc[]

doc/man/pkgctl-repo-pull.1.asciidoc

@@ -0,0 +1,50 @@
+pkgctl-repo-pull(1)
+===================
+
+Name
+----
+pkgctl-repo-pull - Pull in git changes
+
+Synopsis
+--------
+pkgctl repo pull [OPTIONS] [PATH...]
+
+Description
+-----------
+
+Update package repositories from their git remotes.
+
+If only a single package is given the command also automatically shows a diff
+of what has changed since the last time the repository was updated.
+
+Options
+-------
+
+*--discard-changes*::
+	Discard changes if the index or working tree was modified. Otherwise abort
+	if anything was modified.
+
+*--autostash*::
+	Stash before pulling and unstash afterwards
+
+*--quiet*::
+	Do not show any longer terminal output like diffs.
+
+*--show-diff*::
+	Show what has changed since the last time the repository was updated. Is
+	automatically set when only one 'PATH' is given.
+
+*-j, --jobs* 'N'::
+	Run up to N jobs in parallel. By default the number of jobs is equal to the
+	number of available processing units. For sequential processing this option
+	needs to be passed with 1.
+
+*-h, --help*::
+	Show a help text
+
+See Also
+--------
+
+git-pull(1)
+
+include::include/footer.asciidoc[]

doc/man/pkgctl-repo.1.asciidoc

@@ -44,6 +44,9 @@ pkgctl repo configure::
 pkgctl repo create::
 	Create a new GitLab package repository
 
+pkgctl repo pull::
+	Update package repositories from their git remotes
+
 pkgctl repo switch::
 	Switch a package repository to a specified version
 
@@ -57,6 +60,7 @@ pkgctl-repo-clean(1)
 pkgctl-repo-clone(1)
 pkgctl-repo-configure(1)
 pkgctl-repo-create(1)
+pkgctl-repo-pull(1)
 pkgctl-repo-switch(1)
 pkgctl-repo-web(1)
 

src/arch-nspawn.in

@@ -112,7 +112,13 @@ copy_hostconf () {
 	[[ -n $host_cachemirrors ]] && printf 'CacheServer = %s\n' "${host_cachemirrors[@]}" >>"$working_dir/etc/pacman.d/mirrorlist"
 
 	[[ -n $pac_conf ]] && cp "$pac_conf" "$working_dir/etc/pacman.conf"
-	[[ -n $makepkg_conf ]] && cp "$makepkg_conf" "$working_dir/etc/makepkg.conf"
+	if [[ -n $makepkg_conf ]]; then
+		cp "$makepkg_conf" "$working_dir/etc/makepkg.conf"
+		if [[ -d "${makepkg_conf}.d" ]] && is_globfile "${makepkg_conf}.d"/*.conf; then
+			mkdir --parents "$working_dir/etc/makepkg.conf.d/"
+			cp "${makepkg_conf}.d/"*.conf "$working_dir/etc/makepkg.conf.d/"
+		fi
+	fi
 
 	local file
 	for file in "${files[@]}"; do

src/archbuild.in

@@ -79,6 +79,13 @@ check_root SOURCE_DATE_EPOCH,SRCDEST,SRCPKGDEST,PKGDEST,LOGDEST,MAKEFLAGS,PACKAG
 # Pass all arguments after -- right to makepkg
 makechrootpkg_args+=("${@:$OPTIND}")
 
+# Automatically recreate the root chroot if a version mismatch is detected
+CURRENT_CHROOT_VERSION=$(cat "${chroots}/${repo}-${arch}/root/.arch-chroot")
+if [[ -f "${chroots}/${repo}-${arch}/root/.arch-chroot" ]] && [[ "$CURRENT_CHROOT_VERSION" != "$CHROOT_VERSION" ]]; then
+	warning "Recreating chroot '%s' (%s) as it is not at version %s" "${chroots}/${repo}-${arch}/root" "$CURRENT_CHROOT_VERSION" "$CHROOT_VERSION"
+	clean_first=true
+fi
+
 if ${clean_first} || [[ ! -d "${chroots}/${repo}-${arch}" ]]; then
 	msg "Creating chroot for [%s] (%s)..." "${repo}" "${arch}"
 

src/lib/archroot.sh

@@ -4,7 +4,7 @@
 :
 
 # shellcheck disable=2034
-CHROOT_VERSION='v5'
+CHROOT_VERSION='v6'
 
 ##
 #  usage : check_root $keepenv

src/lib/build/build.sh

@@ -8,8 +8,6 @@ DEVTOOLS_INCLUDE_BUILD_SH=1
 _DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@}
 # shellcheck source=src/lib/common.sh
 source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh
-# shellcheck source=src/lib/build/offload.sh
-source "${_DEVTOOLS_LIBRARY_DIR}"/lib/build/offload.sh
 # shellcheck source=src/lib/db/update.sh
 source "${_DEVTOOLS_LIBRARY_DIR}"/lib/db/update.sh
 # shellcheck source=src/lib/release.sh
@@ -465,7 +463,7 @@ pkgctl_build() {
 			fi
 
 			if (( OFFLOAD )); then
-				pkgctl_build_offload_client "${pkgbase}" "${pkgrepo}" "${arch}" "${BUILD_OPTIONS[@]}" -- "${MAKECHROOT_OPTIONS[@]}" -l "${WORKER}" -- "${MAKEPKG_OPTIONS[@]}"
+				offload-build --repo "${pkgrepo}" -- "${BUILD_OPTIONS[@]}" -- "${MAKECHROOT_OPTIONS[@]}" -l "${WORKER}" -- "${MAKEPKG_OPTIONS[@]}"
 			else
 				"${BUILDTOOL}" "${BUILD_OPTIONS[@]}" -- "${MAKECHROOT_OPTIONS[@]}" -l "${WORKER}" -- "${MAKEPKG_OPTIONS[@]}"
 			fi
@@ -483,9 +481,6 @@ pkgctl_build() {
 		# shellcheck disable=SC2119
 		write_srcinfo_file
 
-		version=$(get_full_version)
-		msg "Finished building %s %s" "${pkgbase}" "${version}"
-
 		# test-install (some of) the produced packages
 		if [[ ${INSTALL_TO_HOST} == auto ]] || [[ ${INSTALL_TO_HOST} == all ]]; then
 			# shellcheck disable=2119

src/lib/build/offload.sh

@@ -1,228 +0,0 @@
-#!/hint/bash
-
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-[[ -z ${DEVTOOLS_INCLUDE_BUILD_OFFLOAD_SH:-} ]] || return 0
-DEVTOOLS_INCLUDE_BUILD_OFFLOAD_SH=1
-
-_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@}
-# shellcheck source=src/lib/common.sh
-source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh
-# shellcheck source=src/lib/util/makepkg.sh
-source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/makepkg.sh
-
-source /usr/share/makepkg/util/config.sh
-source /usr/share/makepkg/util/message.sh
-
-set -eo pipefail
-
-
-PKGCTL_OFFLOAD_CACHE_HOME="${XDG_CACHE_HOME:-$HOME/.cache}/pkgctl/offload"
-
-pkgctl_build_offload_usage() {
-	local -r COMMAND=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}}
-	cat <<- _EOF_
-		Usage: ${COMMAND} [COMMAND] [OPTIONS]...
-
-		Server commands to build packages remotely by offloading the job.
-
-		For internal use only!
-_EOF_
-}
-
-pkgctl_build_offload() {
-	if (( $# < 1 )); then
-		pkgctl_build_offload_usage
-		exit 1
-	fi
-
-	while (( $# )); do
-		case $1 in
-			-h|--help)
-				pkgctl_build_offload_usage
-				exit 0
-				;;
-			create-builddir)
-				shift
-				pkgctl_build_offload_server_create_builddir "$@"
-				exit 0
-				;;
-			clean-builddir)
-				shift
-				pkgctl_build_offload_server_clean_builddir "$@"
-				exit 0
-				;;
-			build)
-				shift
-				pkgctl_build_offload_server_build "$@"
-				exit 0
-				;;
-			collect-files)
-				shift
-				pkgctl_build_offload_server_collect_files "$@"
-				exit 0
-				;;
-			collect-logs)
-				shift
-				pkgctl_build_offload_server_collect_logs "$@"
-				exit 0
-				;;
-			*)
-				die "invalid argument: %s" "$1"
-				;;
-		esac
-	done
-}
-
-pkgctl_build_offload_client() {
-	local pkgbase=$1
-	local pkgrepo=$2
-	local pkgarch=$3
-	shift 3
-	local server=build.archlinux.org
-	# shellcheck disable=SC2031
-	local working_dir=$PWD
-	local _srcpkg srcpkg files
-
-	[[ -z ${WORKDIR:-} ]] && setup_workdir
-	TEMPDIR=$(mktemp --tmpdir="${WORKDIR}" --directory "offload.${pkgbase}.${pkgrepo}-${pkgarch}XXXXXXXXXX")
-
-	# Load makepkg.conf variables to be available
-	# shellcheck disable=SC2119
-	load_makepkg_config
-
-	# Use a source-only tarball as an intermediate to transfer files. This
-	# guarantees the checksums are okay, and guarantees that all needed files are
-	# transferred, including local sources, install scripts, and changelogs.
-	export SRCPKGDEST="${TEMPDIR}"
-	if ! makepkg_source_package; then
-		die "unable to make source package"
-		return 1
-	fi
-
-	# Temporary cosmetic workaround makepkg if SRCDEST is set somewhere else
-	# but an empty src dir is created in PWD. Remove once fixed in makepkg.
-	rmdir --ignore-fail-on-non-empty src 2>/dev/null || true
-
-	local builddir
-	builddir=$(
-		ssh "${SSH_OPTS[@]}" -- "$server" pkgctl offload create-builddir "${pkgbase@Q}" "${pkgrepo@Q}" "${pkgarch@Q}"
-	)
-
-	# Transfer the srcpkg to the server
-	msg "Transferring source package to the server..."
-	_srcpkg=("$SRCPKGDEST"/*"$SRCEXT")
-	srcpkg="${_srcpkg[0]}"
-	if ! rsync "${RSYNC_OPTS[@]}" -- "$srcpkg" "$server":"${builddir}"; then
-		die "failed to rsync sources to offload server"
-		return 1
-	fi
-
-	# Execute build
-	if ssh "${SSH_OPTS[@]}" -t -- "$server" pkgctl offload build "${builddir@Q}" "${srcpkg@Q}" "${pkgrepo@Q}" "${pkgarch@Q}" "${@@Q}"; then
-		# Get an array of files that should be downloaded from the server
-		mapfile -t files < <(
-			ssh "${SSH_OPTS[@]}" -- "$server" pkgctl offload collect-files "${builddir@Q}" "${pkgrepo@Q}" "${pkgarch@Q}"
-		)
-	else
-		# Build failed, only the logs should be downloaded from the server
-		mapfile -t files < <(
-			ssh "${SSH_OPTS[@]}" -- "$server" pkgctl offload collect-logs "${builddir@Q}"
-		)
-	fi
-
-	# Check if we collected any files to download
-	if (( ${#files[@]} == 0 )); then
-		die "failed to collect files to download"
-		return 1
-	fi
-
-	msg 'Downloading files...'
-	rsync "${RSYNC_OPTS[@]}" -- "${files[@]/#/$server:}" "${TEMPDIR}/"
-
-	# Clean remote build dir
-	ssh "${SSH_OPTS[@]}" -- "$server" pkgctl offload clean-builddir "${builddir@Q}"
-
-	# Move all log files to LOGDEST
-	if is_globfile "${TEMPDIR}"/*.log; then
-		mv "${TEMPDIR}"/*.log "${LOGDEST:-${working_dir}}/"
-	fi
-
-	# Assume build failed if we didn't download any package files
-	if ! is_globfile "${TEMPDIR}"/*.pkg.tar*; then
-		error "Build failed, check logs in ${LOGDEST:-${working_dir}}"
-		return 1
-	fi
-
-	# Building a package may change the PKGBUILD during update_pkgver
-	mv "${TEMPDIR}/PKGBUILD" "${working_dir}/"
-	mv "${TEMPDIR}"/*.pkg.tar* "${PKGDEST:-${working_dir}}/"
-	return 0
-}
-
-pkgctl_build_offload_server_build() {
-	local builddir=$1
-	local srcpkg=$2
-	local pkgrepo=$3
-	local pkgarch=$4
-	shift 4
-	local buildtool
-
-	if [[ -n $pkgarch ]]; then
-		buildtool="${pkgrepo}-${pkgarch}-build"
-	else
-		buildtool="${pkgrepo}-build"
-	fi
-
-	cd "${builddir}"
-	bsdtar --strip-components 1 -xvf "$(basename "$srcpkg")"
-	LOGDEST="" "${buildtool}" "$@"
-}
-
-pkgctl_build_offload_server_create_builddir() {
-	local pkgbase=$1
-	local pkgrepo=$2
-	local pkgarch=$3
-	mkdir --parents "${PKGCTL_OFFLOAD_CACHE_HOME}"
-	mktemp --directory --tmpdir="${PKGCTL_OFFLOAD_CACHE_HOME}" "${pkgbase}.${pkgrepo}-${pkgarch}XXXXXXXXXX"
-}
-
-pkgctl_build_offload_server_clean_builddir() {
-	local builddir=$1
-	rm --recursive --force -- "${builddir}"
-}
-
-pkgctl_build_offload_server_collect_files() {
-	local builddir=$1
-	local pkgrepo=$2
-	local pkgarch=$3
-
-	local makepkg_config
-	local makepkg_user_config
-
-	# fallback config for multilib
-	if [[ ${pkgrepo} == multilib* ]] && [[ -z ${pkgarch} ]]; then
-		pkgarch=x86_64
-	fi
-
-	cd "${builddir}"
-	makepkg_user_config="${XDG_CONFIG_HOME:-$HOME/.config}/pacman/makepkg.conf"
-	makepkg_config="${_DEVTOOLS_LIBRARY_DIR}/makepkg.conf.d/${pkgarch}.conf"
-	if [[ -f ${_DEVTOOLS_LIBRARY_DIR}/makepkg.conf.d/${pkgrepo}-${pkgarch}.conf ]]; then
-		makepkg_config="${_DEVTOOLS_LIBRARY_DIR}/makepkg.conf.d/${pkgrepo}-${pkgarch}.conf"
-	fi
-	while read -r file; do
-		if [[ -f "${file}" ]]; then
-			printf "%s\n" "${file}"
-		fi
-	done < <(makepkg --config <(cat "${makepkg_user_config}" "${makepkg_config}" 2>/dev/null) --packagelist)
-
-	printf "%s\n" "${builddir}/PKGBUILD"
-
-	pkgctl_build_offload_server_collect_logs "${builddir}"
-}
-
-pkgctl_build_offload_server_collect_logs() {
-	local builddir=$1
-	find "${builddir}" -name "*.log"
-}

src/lib/repo.sh

@@ -31,6 +31,7 @@ pkgctl_repo_usage() {
 		    clone          Clone a package repository
 		    configure      Configure a clone according to distro specs
 		    create         Create a new GitLab package repository
+		    pull           Update package repositories from their git remote
 		    switch         Switch a package repository to a specified version
 		    web            Open the packaging repository's website
 
@@ -40,6 +41,7 @@ pkgctl_repo_usage() {
 		EXAMPLES
 		    $ ${COMMAND} clean --interactive *
 		    $ ${COMMAND} clone libfoo linux libbar
+		    $ ${COMMAND} pull libfoo linux libbar
 		    $ ${COMMAND} clone --maintainer mynickname
 		    $ ${COMMAND} configure *
 		    $ ${COMMAND} create libfoo
@@ -101,6 +103,14 @@ pkgctl_repo() {
 				pkgctl_repo_switch "$@"
 				exit 0
 				;;
+			pull)
+				_DEVTOOLS_COMMAND+=" $1"
+				shift
+				# shellcheck source=src/lib/repo/pull.sh
+				source "${_DEVTOOLS_LIBRARY_DIR}"/lib/repo/pull.sh
+				pkgctl_repo_pull "$@"
+				exit 0
+				;;
 			web)
 				_DEVTOOLS_COMMAND+=" $1"
 				shift

src/lib/repo/pull.sh

@@ -0,0 +1,195 @@
+#!/bin/bash
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+[[ -z ${DEVTOOLS_INCLUDE_REPO_PULL_SH:-} ]] || return 0
+DEVTOOLS_INCLUDE_REPO_PULL_SH=1
+
+_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@}
+# shellcheck source=src/lib/common.sh
+source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh
+# shellcheck source=src/lib/util/git.sh
+source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/git.sh
+
+source /usr/share/makepkg/util/message.sh
+
+set -eo pipefail
+
+
+pkgctl_repo_pull_usage() {
+	local -r COMMAND=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}}
+	cat <<- _EOF_
+		Usage: ${COMMAND} [OPTIONS] [PKGBASE]...
+
+		Update package repositories from their git remotes.
+
+		OPTIONS
+		    --discard-changes   Discard changes if index or working tree is dirty
+		    --show-diff         Always enable showing the diff
+		    --autostash         Stash before pulling and unstash afterwards
+		    -j, --jobs N        Run up to N jobs in parallel (default: $(nproc))
+		    --quiet             Disable printing longer terminal output
+		    -h, --help          Show this help text
+
+		EXAMPLES
+		    $ ${COMMAND} gopass gopass-jsonapi
+_EOF_
+}
+
+pkgctl_repo_pull() {
+	# options
+	local paths path pkgbase branch jobs remote
+	jobs=$(nproc)
+	local command=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}}
+	local git_rebase_options=()
+	local SHOW_DIFF=0
+	local QUIET=0
+	local DISCARD_CHANGES=0
+	local AUTOSTASH=0
+
+	while (( $# )); do
+		case $1 in
+			-h|--help)
+				pkgctl_repo_pull_usage
+				exit 0
+				;;
+			--discard-changes)
+				DISCARD_CHANGES=1
+				shift
+				;;
+			--show-diff)
+				SHOW_DIFF=1
+				shift
+				;;
+			--quiet)
+				QUIET=1
+				shift
+				;;
+			--autostash)
+				AUTOSTASH=1
+				git_rebase_options+=(--autostash)
+				shift
+				;;
+			-j|--jobs)
+				(( $# <= 1 )) && die "missing argument for %s" "$1"
+				jobs=$2
+				shift 2
+				;;
+			--)
+				shift
+				break
+				;;
+			-*)
+				die "invalid argument: %s" "$1"
+				;;
+			*)
+				paths=("$@")
+				break
+				;;
+		esac
+	done
+
+	# check if invoked without any path from within a packaging repo
+	if (( ${#paths[@]} == 0 )); then
+		paths=(".")
+	fi
+
+	# check if we are only working on one repo and enable
+	if (( ${#paths[@]} == 1 )) && (( ! QUIET )); then
+		SHOW_DIFF=1
+	fi
+
+	# parallelization
+	if [[ ${jobs} != 1 ]] && (( ${#paths[@]} > 1 )); then
+		if [[ -n ${BOLD} ]]; then
+			export DEVTOOLS_COLOR=always
+		fi
+
+		# warm up ssh connection as it may require user input (key unlock, hostkey verification etc)
+		git_warmup_ssh_connection
+
+		# disable diffs if not enabled explicitly
+		if (( ! SHOW_DIFF )); then
+			command+=" --quiet"
+		fi
+
+		if (( DISCARD_CHANGES )); then
+			command+=" --discard-changes"
+		fi
+
+		if (( AUTOSTASH )); then
+			command+=" --autostash"
+		fi
+
+		if ! parallel --bar --jobs "${jobs}" "${command}" ::: "${paths[@]}"; then
+			die 'Failed to pull some packages, please check the previous output'
+		fi
+		exit 0
+	fi
+
+	for path in "${paths[@]}"; do
+		# skip paths that are not directories
+		if [[ ! -d "${path}" ]]; then
+			continue
+		fi
+
+		if [[ ! -f "${path}/PKGBUILD" ]]; then
+			msg_error "  Not a package repository: ${path}"
+			continue
+		fi
+
+		if [[ ! -d "${path}/.git" ]]; then
+			msg_error "  Not a Git repository: ${path}"
+			continue
+		fi
+
+		pkgbase=$(basename "$(realpath "${path}")")
+		pkgbase=${pkgbase%.git}
+		msg "Updating ${pkgbase}"
+
+		branch=$(git -C "${path}" symbolic-ref --quiet --short HEAD)
+		if [[ ${branch} != main ]]; then
+			msg_warn "  Current branch is ${branch}, not updating from canonical upstream: ${pkgbase}"
+		fi
+
+		if ! git -C "${path}" diff-files --quiet && (( ! AUTOSTASH )) && (( ! DISCARD_CHANGES )); then
+			msg_error "  Index contains unstaged changes, please stash them or pass --autostash before pulling: ${pkgbase}"
+			continue
+		fi
+
+		if ! git -C "${path}" diff-index --cached --quiet HEAD && (( ! AUTOSTASH )) && (( ! DISCARD_CHANGES )); then
+			msg_error "  Index contains uncommited changes, please commit or stash them before pulling: ${pkgbase}"
+			continue
+		fi
+
+		remote=$(git -C "${path}" config "branch.${branch}.remote")
+		if [[ -z "${remote}" ]]; then
+			msg_error "  No upstream tracking branch configured: ${pkgbase}"
+			continue
+		fi
+
+		if ! git -C "${path}" fetch --quiet "${remote}"; then
+			msg_error "  Error while fetching: ${pkgbase}"
+			continue
+		fi
+
+		if [[ $(git -C "${path}" rev-parse HEAD) == $(git -C "${path}" rev-parse FETCH_HEAD) ]]; then
+			msg2 "Repo is up to date, nothing to do"
+			continue
+		fi
+
+		# discard any local modifications
+		if (( DISCARD_CHANGES )); then
+			git -C "${path}" restore --staged --worktree -- .
+		fi
+
+		if (( SHOW_DIFF )) && (( ! QUIET )); then
+			git -C "${path}" --no-pager diff --color --patch-with-stat HEAD..FETCH_HEAD
+		fi
+
+		if ! git -C "${path}" rebase --quiet "${git_rebase_options[@]}" "${remote}/${branch}"; then
+			msg_error "  Error while pulling in the changes for ${pkgbase}"
+			exit 1
+		fi
+	done
+}

src/makerepropkg.in

@@ -93,14 +93,36 @@ get_makepkg_conf() {
     local fname=${1}
     local arch="${2}"
     local makepkg_conf="${3}"
+
     if ! buildtool_file=$(get_pkgfile "${fname}"); then
-        error "failed to retrieve ${fname}"
+       error "failed to retrieve ${fname}"
+       return 1
+    fi
+    buildtool_file="${buildtool_file/file:\/\//}"
+    msg "using makepkg.conf from ${fname}"
+
+    # try to handle config of legacy devtools
+    if bsdtar --list --file "${buildtool_file}" "usr/share/devtools/makepkg-${arch}.conf" &>/dev/null; then
+        bsdtar --extract --to-stdout --fast-read --file "${buildtool_file}" "usr/share/devtools/makepkg-${arch}.conf" > "${makepkg_conf}"
+        return $?
+    fi
+
+    msg2 "extracting ${arch}.conf from devtools archive"
+    if ! bsdtar --extract --to-stdout --fast-read --file "${buildtool_file}" "usr/share/devtools/makepkg.conf.d/${arch}.conf" > "${makepkg_conf}"; then
+        error "failed to extract 'usr/share/devtools/makepkg.conf.d/${arch}.conf' from devtools archive"
         return 1
     fi
-    msg2 "using makepkg.conf from ${fname}"
-	if ! bsdtar xOqf "${buildtool_file/file:\/\//}" "usr/share/devtools/makepkg.conf.d/${arch}.conf" > "${makepkg_conf}"; then
-		bsdtar xOqf "${buildtool_file/file:\/\//}" "usr/share/devtools/makepkg-${arch}.conf" > "${makepkg_conf}"
-	fi
+
+    mkdir --parents "${makepkg_conf}.d"
+    if bsdtar --list --file "${buildtool_file}" "usr/share/devtools/makepkg.conf.d/conf.d" &>/dev/null; then
+        msg2 "extracting conf.d from devtools archive"
+        bsdtar --extract --file "${buildtool_file}" --cd "${makepkg_conf}.d" --strip-components 4 "usr/share/devtools/makepkg.conf.d/conf.d"
+    fi
+    if bsdtar --list --file "${buildtool_file}" "usr/share/devtools/makepkg.conf.d/${arch}.conf.d" &>/dev/null; then
+        msg2 "extracting ${arch}.conf.d from devtools archive"
+        bsdtar --extract --file "${buildtool_file}" --cd "${makepkg_conf}.d" --strip-components 4 "usr/share/devtools/makepkg.conf.d/${arch}.conf.d"
+    fi
+
     return 0
 }
 
@@ -186,7 +208,7 @@ for f in "${splitpkgs[@]}"; do
 done
 
 if (( ${#cache_dirs[@]} == 0 )); then
-	mapfile -t cache_dirs < <(pacman-conf CacheDir)
+    mapfile -t cache_dirs < <(pacman-conf CacheDir)
 fi
 
 ORIG_HOME=${HOME}

src/offload-build.in

@@ -0,0 +1,162 @@
+#!/bin/bash
+#
+# offload-build - build a PKGBUILD on a remote server using makechrootpkg.
+#
+# Copyright (c) 2019 by Eli Schwartz <eschwartz@archlinux.org>
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@}
+# shellcheck source=src/lib/common.sh
+source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh
+# shellcheck source=src/lib/util/makepkg.sh
+source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/makepkg.sh
+
+source /usr/share/makepkg/util/config.sh
+
+# Deprecation warning
+if [[ -z $_DEVTOOLS_COMMAND ]]; then
+	warning "${0##*/} is deprecated and will be removed. Use 'pkgctl build --offload' instead"
+fi
+
+# global defaults suitable for use by Arch staff
+repo=extra
+arch=x86_64
+server=build.archlinux.org
+
+usage() {
+    cat <<- _EOF_
+		Usage: ${BASH_SOURCE[0]##*/} [--repo REPO] [--arch ARCHITECTURE] [--server SERVER] -- [ARCHBUILD_ARGS]
+
+		Build a PKGBUILD on a remote server using makechrootpkg. Requires a remote user
+		that can run archbuild without password auth. Options passed after a -- are
+		passed on to archbuild, and eventually to makechrootpkg.
+
+		OPTIONS
+		    -r, --repo      Build against a specific repository (current: $repo)
+		    -a, --arch      Build against a specific architecture (current: $arch)
+		    -s, --server    Offload to a specific build server (current: $server)
+		    -h, --help      Show this help text
+_EOF_
+}
+
+# option checking
+while (( $# )); do
+    case $1 in
+        -h|--help)
+            usage
+            exit 0
+            ;;
+        -r|--repo)
+            repo=$2
+            shift 2
+            ;;
+        -a|--arch)
+            arch=$2
+            shift 2
+            ;;
+        -s|--server)
+            server=$2
+            shift 2
+            ;;
+        --)
+            shift
+            break
+            ;;
+        *)
+            die "invalid argument: %s" "$1"
+            ;;
+    esac
+done
+
+# multilib must be handled specially
+archbuild_arch="${arch}"
+if [[ $repo = multilib* ]]; then
+    archbuild_arch=
+fi
+
+archbuild_cmd=("${repo}${archbuild_arch:+-$archbuild_arch}-build" "$@")
+
+[[ -z ${WORKDIR:-} ]] && setup_workdir
+export TEMPDIR=$(mktemp --tmpdir="${WORKDIR}" --directory offload-build.XXXXXXXXXX)
+
+# Load makepkg.conf variables to be available
+load_makepkg_config
+
+# Use a source-only tarball as an intermediate to transfer files. This
+# guarantees the checksums are okay, and guarantees that all needed files are
+# transferred, including local sources, install scripts, and changelogs.
+export SRCPKGDEST="${TEMPDIR}"
+makepkg_source_package || die "unable to make source package"
+
+# Temporary cosmetic workaround makepkg if SRCDEST is set somewhere else
+# but an empty src dir is created in PWD. Remove once fixed in makepkg.
+rmdir --ignore-fail-on-non-empty src 2>/dev/null || true
+
+# Create a temporary directory on the server
+remote_temp=$(
+    ssh "${SSH_OPTS[@]}" -- "$server" '
+        temp="${XDG_CACHE_HOME:-$HOME/.cache}/offload-build" &&
+        mkdir -p "$temp" &&
+        mktemp --directory --tmpdir="$temp"
+')
+
+# Transfer the srcpkg to the server
+msg "Transferring source package to the server..."
+_srcpkg=("$SRCPKGDEST"/*"$SRCEXT")
+srcpkg="${_srcpkg[0]}"
+rsync "${RSYNC_OPTS[@]}" -- "$srcpkg" "$server":"$remote_temp" || die
+
+# Prepare the srcpkg on the server
+msg "Extracting srcpkg"
+ssh "${SSH_OPTS[@]}" -- "$server" "cd ${remote_temp@Q} && bsdtar --strip-components 1 -xvf $(basename "$srcpkg")" || die
+
+# Run the build command on the server
+msg "Running archbuild"
+# shellcheck disable=SC2145
+if ssh "${SSH_OPTS[@]}" -t -- "$server" "cd ${remote_temp@Q} && export LOGDEST="" && ${archbuild_cmd[@]@Q}"; then
+    msg "Build complete"
+
+    # Get an array of files that should be downloaded from the server
+    mapfile -t files < <(
+        ssh "${SSH_OPTS[@]}" -- "$server" "
+            cd ${remote_temp@Q}"' &&
+            makepkg_user_config="${XDG_CONFIG_HOME:-$HOME/.config}/pacman/makepkg.conf" &&
+            makepkg_config="/usr/share/devtools/makepkg.conf.d/'"${arch}"'.conf" &&
+            if [[ -f /usr/share/devtools/makepkg.conf.d/'"${repo}"'-'"${arch}"'.conf ]]; then
+                makepkg_config="/usr/share/devtools/makepkg.conf.d/'"${repo}"'-'"${arch}"'.conf"
+            fi &&
+            while read -r file; do
+                [[ -f "${file}" ]] && printf "%s\n" "${file}" ||:
+            done < <(makepkg --config <(cat "${makepkg_user_config}" "${makepkg_config}" 2>/dev/null) --packagelist) &&
+            printf "%s\n" '"${remote_temp@Q}/PKGBUILD"'
+
+            find '"${remote_temp@Q}"' -name "*.log"
+    ')
+else
+    # Build failed, only the logs should be downloaded from the server
+    mapfile -t files < <(
+        ssh "${SSH_OPTS[@]}" -- "$server" '
+            find '"${remote_temp@Q}"' -name "*.log"
+    ')
+fi
+
+
+if (( ${#files[@]} )); then
+    msg 'Downloading files...'
+    rsync "${RSYNC_OPTS[@]}" -- "${files[@]/#/$server:}" "${TEMPDIR}/" || die
+
+    if is_globfile "${TEMPDIR}"/*.log; then
+        mv "${TEMPDIR}"/*.log "${LOGDEST:-${PWD}}/"
+    fi
+    if is_globfile "${TEMPDIR}"/*.pkg.tar*; then
+        # Building a package may change the PKGBUILD during update_pkgver
+        mv "${TEMPDIR}/PKGBUILD" "${PWD}/"
+        mv "${TEMPDIR}"/*.pkg.tar* "${PKGDEST:-${PWD}}/"
+    else
+        error "Build failed, check logs in ${LOGDEST:-${PWD}}"
+        exit 1
+    fi
+else
+    exit 1
+fi

src/pkgctl.in

@@ -113,14 +113,6 @@ while (( $# )); do
 			pkgctl_issue "$@"
 			exit 0
 			;;
-		offload)
-			_DEVTOOLS_COMMAND+=" $1"
-			shift
-			# shellcheck source=src/lib/build/offload.sh
-			source "${_DEVTOOLS_LIBRARY_DIR}"/lib/build/offload.sh
-			pkgctl_build_offload "$@"
-			exit 0
-			;;
 		release)
 			_DEVTOOLS_COMMAND+=" $1"
 			shift