offload-build: support different pkgdatadir locations on the remote

Use devtools-config to get the pkgdatadir on the remote and use that
location to search for the appropriate makepkg.conf file. This way the
remote environment can have a non repo matching devtools build for
example being installed to /usr/local

.github/workflows/test.yml

@@ -1,15 +0,0 @@
-on: push
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    container: archlinux/archlinux:latest
-    steps:
-      - name: Install dependencies
-        run: pacman -Syu --noconfirm m4 make openssh subversion rsync arch-install-scripts git bzr mercurial diffutils asciidoc shellcheck
-      - uses: actions/checkout@v2
-      - name: Run tests
-        run: |
-           make PREFIX=/usr
-           make PREFIX=/usr DESTDIR="$(mktemp -d)" install
-           make check || true
-           SHELLCHECK_OPTS="-S error" make check

.gitignore

@@ -6,6 +6,8 @@ archrelease
 bash_completion
 checkpkg
 commitpkg
+devtools-config
+diffpkg
 finddeps
 lddd
 makechrootpkg

.gitlab-ci.yml

@@ -0,0 +1,21 @@
+image: "archlinux:latest"
+
+stages:
+  - build
+  - test
+
+build:
+  stage: build
+  needs: []
+  script:
+    - pacman -Syu --noconfirm m4 make openssh subversion rsync arch-install-scripts git bzr mercurial diffutils asciidoc
+    - make PREFIX=/usr
+    - make PREFIX=/usr DESTDIR=build install
+
+check:
+  stage: test
+  needs: []
+  script:
+    - pacman -Syu --noconfirm m4 make openssh subversion rsync arch-install-scripts git bzr mercurial diffutils asciidoc shellcheck
+    - make check || true
+    - SHELLCHECK_OPTS="-S error" make check

Makefile

@@ -1,4 +1,4 @@
-V=20220126
+V=20220207
 BUILDTOOLVER ?= $(V)
 
 PREFIX = /usr/local
@@ -12,6 +12,8 @@ IN_PROGS = \
 	checkpkg \
 	commitpkg \
 	crossrepomove\
+	devtools-config \
+	diffpkg \
 	finddeps \
 	find-libdeps \
 	lddd \
@@ -76,6 +78,8 @@ MANS = \
 	doc/makechrootpkg.1 \
 	doc/lddd.1 \
 	doc/checkpkg.1 \
+	doc/devtools-config.1 \
+	doc/diffpkg.1 \
 	doc/offload-build.1 \
 	doc/sogrep.1 \
 	doc/makerepropkg.1 \
@@ -146,7 +150,7 @@ tag:
 	@git tag --sign --message "Version $(TODAY)" $(TODAY)
 
 dist:
-	git archive --format=tar --prefix=devtools-$(V)/ $(V) | gzip -9 > devtools-$(V).tar.gz
+	git archive --format=tar --prefix=devtools-$(V)/ $(V) | gzip > devtools-$(V).tar.gz
 	gpg --detach-sign --use-agent devtools-$(V).tar.gz
 
 upload:

devtools-config.in

@@ -0,0 +1,46 @@
+#!/bin/bash
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+pkgdatadir='@pkgdatadir@'
+
+# shellcheck disable=SC2059
+die() { printf "error: $1\n" "${@:2}" >&2; exit 1; }
+
+usage() {
+	cat <<- _EOF_
+		Usage: ${BASH_SOURCE[0]##*/} [OPTION...]
+
+		Manage and print environment specific devtools information.
+
+		OPTIONS
+		    --pkgdatadir    Print pkgdatadir location
+		    -h, --help      Show this help text
+_EOF_
+}
+
+print_pkgdatadir() {
+	printf "%s\n" "${pkgdatadir}"
+}
+
+if (( $# == 0 )); then
+	usage
+	exit 0
+fi
+
+# option checking
+while (( $# )); do
+	case $1 in
+		-h|--help)
+			usage
+			exit 0
+			;;
+		--pkgdatadir)
+			print_pkgdatadir
+			exit 0
+			;;
+		*)
+			die "invalid argument: %s" "$1"
+			;;
+	esac
+done

diffpkg.in

@@ -0,0 +1,228 @@
+#!/bin/bash
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+shopt -s extglob
+
+m4_include(lib/common.sh)
+
+usage() {
+    cat <<- _EOF_
+		Usage: ${BASH_SOURCE[0]##*/} [OPTIONS] [MODES] [FILE|PKGNAME...]
+
+		Searches for a locally built package corresponding to the PKGBUILD, and
+		downloads the last version of that package from the Pacman repositories.
+		It then compares the package archives using different modes while using
+		simple tar content list by default.
+
+		When given one package, use it to diff against the locally built one.
+		When given two packages, diff both packages against each other.
+
+		In either case, a package name will be converted to a filename from the
+		cache, and diffpkg will proceed as though this filename was initially
+		specified.
+
+		OPTIONS
+		    -M, --makepkg-config Set an alternate makepkg configuration file
+		    -v, --verbose        Provide more detailed/unfiltered output
+		    -h, --help           Show this help text
+
+		MODES
+		    -l, --list           Activate content list diff mode (default)
+		    -d, --diffoscope     Activate diffoscope diff mode
+		    -p, --pkginfo        Activate .PKGINFO diff mode
+		    -b, --buildinfo      Activate .BUILDINFO diff mode
+_EOF_
+}
+
+MAKEPKG_CONF=/etc/makepkg.conf
+VERBOSE=0
+TARLIST=0
+DIFFOSCOPE=0
+PKGINFO=0
+BUILDINFO=0
+
+# option checking
+while (( $# )); do
+	case $1 in
+		-h|--help)
+			usage
+			exit 0
+			;;
+		-M|--makepkg-config)
+			MAKEPKG_CONF="$2"
+			shift 2
+			;;
+		-l|--list)
+			TARLIST=1
+			shift
+			;;
+		-d|--diffoscope)
+			DIFFOSCOPE=1
+			shift
+			;;
+		-p|--pkginfo)
+			PKGINFO=1
+			shift
+			;;
+		-b|--buildinfo)
+			BUILDINFO=1
+			shift
+			;;
+		-v|--verbose)
+			VERBOSE=1
+			shift
+			;;
+		--)
+			shift
+			break
+			;;
+		-*,--*)
+			die "invalid argument: %s" "$1"
+			;;
+		*)
+			break
+			;;
+	esac
+done
+
+if ! (( DIFFOSCOPE || TARLIST || PKGINFO || BUILDINFO )); then
+	TARLIST=1
+fi
+
+# Source makepkg.conf; fail if it is not found
+if [[ -r "${MAKEPKG_CONF}" ]]; then
+	# shellcheck source=makepkg-x86_64.conf
+	source "${MAKEPKG_CONF}"
+else
+	die "${MAKEPKG_CONF} not found!"
+fi
+
+# Source user-specific makepkg.conf overrides
+if [[ -r "${XDG_CONFIG_HOME:-$HOME/.config}/pacman/makepkg.conf" ]]; then
+	# shellcheck source=/dev/null
+	source "${XDG_CONFIG_HOME:-$HOME/.config}/pacman/makepkg.conf"
+elif [[ -r "$HOME/.makepkg.conf" ]]; then
+	# shellcheck source=/dev/null
+	source "$HOME/.makepkg.conf"
+fi
+
+STARTDIR=$(pwd)
+trap 'rm -rf $TMPDIR' EXIT INT TERM QUIT
+TMPDIR=$(mktemp -d --tmpdir diffpkg-script.XXXXXXXX)
+export TMPDIR
+
+tar_list() {
+	bsdtar tf "$*" | if (( VERBOSE )); then
+		cat
+	else
+		sed -E 's|^usr/lib/modules/[0-9][^/]+|usr/lib/modules/[…]|g'
+	fi | sort
+}
+
+diff_pkgs() {
+	local oldpkg newpkg
+	oldpkg=$(readlink -m "$1")
+	newpkg=$(readlink -m "$2")
+
+	[[ -f $oldpkg ]] || die "No such file: %s" "${oldpkg}"
+	[[ -f $newpkg ]] || die "No such file: %s" "${newpkg}"
+
+	if (( TARLIST )); then
+		tar_list "$oldpkg" > "$TMPDIR/filelist-old"
+		tar_list "$newpkg" > "$TMPDIR/filelist"
+
+		sdiff -s "$TMPDIR/filelist-old" "$TMPDIR/filelist"
+	fi
+
+	if (( PKGINFO )); then
+		bsdtar xOqf "$oldpkg" .PKGINFO > "$TMPDIR/pkginfo-old"
+		bsdtar xOqf "$newpkg" .PKGINFO > "$TMPDIR/pkginfo"
+
+		sdiff -s "$TMPDIR/pkginfo-old" "$TMPDIR/pkginfo"
+	fi
+
+	if (( BUILDINFO )); then
+		bsdtar xOqf "$oldpkg" .BUILDINFO > "$TMPDIR/buildinfo-old"
+		bsdtar xOqf "$newpkg" .BUILDINFO > "$TMPDIR/buildinfo"
+
+		sdiff -s "$TMPDIR/buildinfo-old" "$TMPDIR/buildinfo"
+	fi
+
+	if (( DIFFOSCOPE )); then
+		diffoscope "$oldpkg" "$newpkg"
+	fi
+}
+
+fetch_pkg() {
+	local pkg pkgdest pkgurl
+	case $1 in
+		*://*)
+			pkgurl=$1 ;;
+		/*|*/*)
+			pkgurl=$(readlink -m "$1") ;;
+		*.pkg.tar*)
+			pkgurl=$1 ;;
+		'')
+			;;
+		*)
+			pkg=$1 ;;
+	esac
+
+	[[ -n $pkgurl ]] || pkgurl=$(pacman -Spdd --print-format '%l' --noconfirm "$pkg") ||
+		die "Couldn't download previous package for %s." "$pkg"
+
+	pkg=${pkgurl##*/}
+	pkgdest=$(mktemp -t -d "${pkg}-XXXXXX")/${pkg}
+
+	if [[ $pkgurl = file://* || ( $pkgurl = /* && -f $pkgurl ) ]]; then
+		ln -sf "${pkgurl#file://}" "$pkgdest"
+	elif [[ -f "$PKGDEST/$pkg" ]]; then
+		ln -sf "$PKGDEST/$pkg" "$pkgdest"
+	elif [[ -f "$STARTDIR/$pkg" ]]; then
+		ln -sf "$STARTDIR/$pkg" "$pkgdest"
+	elif [[ $pkgurl = *://* ]]; then
+		curl -fsLC - --retry 3 --retry-delay 3 -o "$pkgdest" "$pkgurl" || \
+			die "Couldn't download %s" "$pkgurl"
+	else
+		die "File not found: %s" "$pkgurl"
+	fi
+
+	echo "$pkgdest"
+}
+
+if (( $# < 2 )); then
+	if [[ ! -f PKGBUILD ]]; then
+		die "This must be run in the directory of a built package.\nTry '$(basename "$0") --help' for more information."
+	fi
+
+	# shellcheck source=PKGBUILD.proto
+	. ./PKGBUILD
+	if [[ ${arch[0]} == 'any' ]]; then
+		CARCH='any'
+	fi
+
+	for _pkgname in "${pkgname[@]}"; do
+		comparepkg=$_pkgname
+		pkgurl=
+		target_pkgver=$(get_full_version "$_pkgname")
+		if ! pkgfile=$(find_cached_package "$_pkgname" "$target_pkgver" "$CARCH"); then
+			die 'tarball not found for package: %s' "${_pkgname}-$target_pkgver"
+		fi
+
+		ln -s "$pkgfile" "$TMPDIR"
+
+		if (( $# )); then
+			comparepkg="$1"
+		fi
+
+		oldpkg=$(fetch_pkg "$comparepkg") || exit 1
+
+		diff_pkgs "$oldpkg" "$pkgfile"
+	done
+else
+	file1=$(fetch_pkg "$1") || exit 1
+	file2=$(fetch_pkg "$2") || exit 1
+
+	diff_pkgs "$file1" "$file2"
+fi

doc/devtools-config.1.asciidoc

@@ -0,0 +1,26 @@
+devtools-config(1)
+================
+
+Name
+----
+devtools-config - Manage and print environment specific devtools information.
+
+Synopsis
+--------
+devtools-config [OPTIONS]
+
+Description
+-----------
+
+Manage and print environment specific devtools information.
+
+Options
+-------
+
+*--pkgdatadir*::
+	Print environment specific pkgdatadir location.
+
+*-h, --help*::
+	Show a help text.
+
+include::footer.asciidoc[]

doc/diffpkg.1.asciidoc

@@ -0,0 +1,57 @@
+diffpkg(1)
+===========
+
+Name
+----
+diffpkg - Compare package files using different modes.
+
+Synopsis
+--------
+diffpkg [OPTIONS] [MODES] [FILE|PKGNAME...]
+
+Description
+-----------
+
+Searches for a locally built package corresponding to the PKGBUILD, and
+downloads the last version of that package from the Pacman repositories.
+It then compares the package archives using different modes while using
+simple tar content list by default.
+
+When given one package, use it to diff against the locally built one.
+When given two packages, diff both packages against each other.
+
+In either case, a package name will be converted to a filename from the
+cache, and diffpkg will proceed as though this filename was initially
+specified.
+
+Options
+-------
+
+*-M, --makepkg-config*::
+	Set an alternate makepkg configuration file
+
+*-v, --verbose*::
+	Provide more detailed/unfiltered output
+
+*-h, --help*::
+	Show a help text
+
+Modes
+-----
+
+*-l, --list*::
+	Activate tar content list diff mode (default)
+
+*-d, --diffoscope*::
+	Activate diffoscope diff mode
+
+*-p, --pkginfo*::
+	Activate .PKGINFO diff mode
+
+*-b, --buildinfo*::
+	Activate .BUILDINFO diff mode
+
+See Also
+--------
+
+include::footer.asciidoc[]

makepkg-x86_64.conf

@@ -45,12 +45,13 @@ CFLAGS="-march=x86-64 -mtune=generic -O2 -pipe -fno-plt -fexceptions \
         -fstack-clash-protection -fcf-protection"
 CXXFLAGS="$CFLAGS -Wp,-D_GLIBCXX_ASSERTIONS"
 LDFLAGS="-Wl,-O1,--sort-common,--as-needed,-z,relro,-z,now"
+LTOFLAGS="-flto=auto"
 #RUSTFLAGS="-C opt-level=2"
 #-- Make Flags: change this for DistCC/SMP systems
 #MAKEFLAGS="-j2"
 #-- Debugging flags
-DEBUG_CFLAGS="-g -fvar-tracking-assignments"
-DEBUG_CXXFLAGS="-g -fvar-tracking-assignments"
+DEBUG_CFLAGS="-g"
+DEBUG_CXXFLAGS="$DEBUG_CFLAGS"
 #DEBUG_RUSTFLAGS="-C debuginfo=2"
 
 #########################################################################

offload-build.in

@@ -101,10 +101,11 @@ mapfile -t files < <(
                 printf "%s\n" "" "-> build complete" &&
                 printf "\t%s\n" "$temp"/*
             } >&2 &&
+            pkgdatadir="$(devtools-config --pkgdatadir||echo /usr/share/devtools)" &&
             makepkg_user_config="${XDG_CONFIG_HOME:-$HOME/.config}/pacman/makepkg.conf" &&
-            makepkg_config="/usr/share/devtools/makepkg-'"${arch}"'.conf" &&
-            if [[ -f /usr/share/devtools/makepkg-'"${repo}"'-'"${arch}"'.conf ]]; then
-                makepkg_config="/usr/share/devtools/makepkg-'"${repo}"'-'"${arch}"'.conf"
+            makepkg_config="${pkgdatadir}/makepkg-'"${arch}"'.conf" &&
+            if [[ -f "${pkgdatadir}/makepkg-'"${repo}"'-'"${arch}"'.conf" ]]; then
+                makepkg_config="${pkgdatadir}/makepkg-'"${repo}"'-'"${arch}"'.conf"
             fi &&
             makepkg --config <(cat "${makepkg_user_config}" "${makepkg_config}" 2>/dev/null) --packagelist &&
             printf "%s\n" "${temp}/PKGBUILD"

sogrep.in

@@ -36,8 +36,11 @@ recache() {
             local dbpath=${SOCACHE_DIR}/${arch}/${repo}.links.tar.gz
             mkdir -p "${dbpath%/*}"
             (( VERBOSE )) && echo "Fetching ${repo}.links.tar.gz..."
-            curl -LR "${verbosity}" -o "${dbpath}" -z "${dbpath}" \
-                "${SOLINKS_MIRROR}/${repo}/os/${arch}/${repo}.links.tar.gz"
+            if ! curl -fLR "${verbosity}" -o "${dbpath}" -z "${dbpath}" \
+                    "${SOLINKS_MIRROR}/${repo}/os/${arch}/${repo}.links.tar.gz"; then
+                echo "error: failed to download links database for repo ${repo}"
+                exit 1
+            fi
         done
     done
 }

zsh_completion.in

@@ -1,4 +1,4 @@
-#compdef archbuild archco arch-nspawn archrelease commitpkg finddeps makechrootpkg mkarchroot rebuildpkgs extrapkg=commitpkg corepkg=commitpkg testingpkg=commitpkg stagingpkg=commitpkg communitypkg=commitpkg community-testingpkg=commitpkg community-stagingpkg=commitpkg multilibpkg=commitpkg multilib-testingpkg=commitpkg extra-x86_64-build=archbuild testing-x86_64-build=archbuild staging-x86_64-build=archbuild multilib-build=archbuild multilib-testing-build=archbuild multilib-staging-build=archbuild kde-unstable-x86_64-build=archbuild gnome-unstable-x86_64-build=archbuild communityco=archco checkpkg sogrep offload-build makerepropkg
+#compdef archbuild archco arch-nspawn archrelease commitpkg devtools-config diffpkg finddeps makechrootpkg mkarchroot rebuildpkgs extrapkg=commitpkg corepkg=commitpkg testingpkg=commitpkg stagingpkg=commitpkg communitypkg=commitpkg community-testingpkg=commitpkg community-stagingpkg=commitpkg multilibpkg=commitpkg multilib-testingpkg=commitpkg extra-x86_64-build=archbuild testing-x86_64-build=archbuild staging-x86_64-build=archbuild multilib-build=archbuild multilib-testing-build=archbuild multilib-staging-build=archbuild kde-unstable-x86_64-build=archbuild gnome-unstable-x86_64-build=archbuild communityco=archco checkpkg sogrep offload-build makerepropkg
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
@@ -41,6 +41,22 @@ _commitpkg_args=(
 	'1:commit_msg'
 )
 
+_devtools_config_args=(
+	'(--pkgdatadir)'{--pkgdatadir}'[Print pkgdatadir location]'
+	'(-h --help)'{-h,--help}'[Display usage]'
+)
+
+_diffpkg_args=(
+	'(-l --list)'{-l,--list}'[Tar content list diff mode]'
+	'(-d --diffoscope)'{-d,--diffoscope}'[Diffoscope diff mode]'
+	'(-p --pkginfo)'{-p,--pkginfo}'[.PKGINFO diff mode]'
+	'(-b --buildinfo)'{-b,--buildinfo}'[.BUILDINFO diff mode]'
+	'(-m --makepkg-config)'{-m,--makepkg-config}'[Location of a makepkg config file]:makepkg_config:_files -g "*.conf(.)"'
+	'(-v --verbose)'{-v,--verbose}'[Provide more detailed/unfiltered output]'
+	'(-h --help)'{-h,--help}'[Display usage]'
+	'*:packages:_devtools_completions_all_packages'
+)
+
 _finddeps_args=(
 	'1:packages:_devtools_completions_all_packages'
 )