Version 20230105

This enables DEBUG_RUSTFLAGS with the appropriate debuginfo settings.

The empty RUSTFLAGS variable is required workaround to avoid double
compilation during package function call. The issue is that the
behavior of the current implementation of buildenv_debugflags is not
idempotent, so consecutive calls will append the same flags again
leading to cargo consider the build inputs to have changed.

Signed-off-by: Levente Polyak <anthraxx@archlinux.org>

Makefile

@@ -1,4 +1,4 @@
-V=20220621
+V=20230105
 BUILDTOOLVER ?= $(V)
 
 PREFIX = /usr/local
@@ -6,6 +6,7 @@ MANDIR = $(PREFIX)/share/man
 BUILDDIR = build
 
 BINPROGS = $(addprefix $(BUILDDIR)/,$(patsubst src/%,bin/%,$(patsubst %.in,%,$(wildcard src/*.in))))
+LIBUTILS = $(wildcard lib/*)
 MAKEPKG_CONFIGS=$(wildcard config/makepkg/*)
 PACMAN_CONFIGS=$(wildcard config/pacman/*)
 SETARCH_ALIASES = $(wildcard config/setarch-aliases.d/*)
@@ -71,13 +72,13 @@ edit = sed -e "s|@pkgdatadir[@]|$(PREFIX)/share/devtools|g"
 GEN_MSG = @echo "GEN $(patsubst $(BUILDDIR)/%,%,$@)"
 
 define buildInScript
-$(1)/%: $(2)%.in
+$(1)/%: $(2)%.in $(LIBUTILS)
 	$$(GEN_MSG)
 	@mkdir -p $$(dir $$@)
 	@$(RM) "$$@"
 	@{ echo -n 'm4_changequote([[[,]]])'; cat $$<; } | m4 -P --define=m4_devtools_version=$$(BUILDTOOLVER) | $(edit) >$$@
 	@chmod $(3) "$$@"
-	@bash -O extglob -n "$$@"
+	@bash -n "$$@"
 endef
 
 $(eval $(call buildInScript,build/bin,src/,555))

config/makepkg/x86_64.conf

@@ -46,13 +46,13 @@ CFLAGS="-march=x86-64 -mtune=generic -O2 -pipe -fno-plt -fexceptions \
 CXXFLAGS="$CFLAGS -Wp,-D_GLIBCXX_ASSERTIONS"
 LDFLAGS="-Wl,-O1,--sort-common,--as-needed,-z,relro,-z,now"
 LTOFLAGS="-flto=auto"
-#RUSTFLAGS="-C opt-level=2"
+RUSTFLAGS=""
 #-- Make Flags: change this for DistCC/SMP systems
 #MAKEFLAGS="-j2"
 #-- Debugging flags
 DEBUG_CFLAGS="-g"
 DEBUG_CXXFLAGS="$DEBUG_CFLAGS"
-#DEBUG_RUSTFLAGS="-C debuginfo=2"
+DEBUG_RUSTFLAGS="-C debuginfo=2"
 
 #########################################################################
 # BUILD ENVIRONMENT
@@ -94,7 +94,7 @@ BUILDENV=(!distcc color !ccache check !sign)
 #-- debug:      Add debugging flags as specified in DEBUG_* variables
 #-- lto:        Add compile flags for building with link time optimization
 #
-OPTIONS=(strip docs !libtool !staticlibs emptydirs zipman purge !debug lto)
+OPTIONS=(strip docs !libtool !staticlibs emptydirs zipman purge debug lto)
 
 #-- File integrity checks to use. Valid: md5, sha1, sha224, sha256, sha384, sha512, b2
 INTEGRITY_CHECK=(sha256)

config/makepkg/x86_64_v3.conf

@@ -46,13 +46,13 @@ CFLAGS="-march=x86-64-v3 -mtune=generic -O2 -pipe -fno-plt -fexceptions \
 CXXFLAGS="$CFLAGS -Wp,-D_GLIBCXX_ASSERTIONS"
 LDFLAGS="-Wl,-O1,--sort-common,--as-needed,-z,relro,-z,now"
 LTOFLAGS="-flto=auto"
-#RUSTFLAGS="-C opt-level=2"
+RUSTFLAGS=""
 #-- Make Flags: change this for DistCC/SMP systems
 #MAKEFLAGS="-j2"
 #-- Debugging flags
 DEBUG_CFLAGS="-g"
 DEBUG_CXXFLAGS="$DEBUG_CFLAGS"
-#DEBUG_RUSTFLAGS="-C debuginfo=2"
+DEBUG_RUSTFLAGS="-C debuginfo=2"
 
 #########################################################################
 # BUILD ENVIRONMENT
@@ -94,7 +94,7 @@ BUILDENV=(!distcc color !ccache check !sign)
 #-- debug:      Add debugging flags as specified in DEBUG_* variables
 #-- lto:        Add compile flags for building with link time optimization
 #
-OPTIONS=(strip docs !libtool !staticlibs emptydirs zipman purge !debug lto)
+OPTIONS=(strip docs !libtool !staticlibs emptydirs zipman purge debug lto)
 
 #-- File integrity checks to use. Valid: md5, sha1, sha224, sha256, sha384, sha512, b2
 INTEGRITY_CHECK=(sha256)

contrib/completion/zsh/_devtools.in

@@ -6,6 +6,7 @@ m4_include(lib/valid-tags.sh)
 m4_include(lib/valid-repos.sh)
 
 _binary_arch=${_arch[*]:0:-1}
+_colors=(never always auto)
 
 _archbuild_args=(
 	'-c[Recreate the chroot before building]'
@@ -47,6 +48,11 @@ _diffpkg_args=(
 	'(-p --pkginfo)'{-p,--pkginfo}'[.PKGINFO diff mode]'
 	'(-b --buildinfo)'{-b,--buildinfo}'[.BUILDINFO diff mode]'
 	'(-m --makepkg-config)'{-m,--makepkg-config}'[Location of a makepkg config file]:makepkg_config:_files -g "*.conf(.)"'
+	'(-u -U --unified)'{-u,-U,--unified}'[Output 3 lines of unified context]'
+	'(-y --side-by-side)'{-y,--side-by-side}'[Output in two columns]'
+	'--color=[Color output]:when:($_colors[*])'
+	'(-W --width=)'{-W,--width=}'[Output at most NUM print columns]:num:(auto columns)'
+	'(-P --pool=)'{-P,--pool=}'[pool directory]:dir:_files -/'
 	'(-v --verbose)'{-v,--verbose}'[Provide more detailed/unfiltered output]'
 	'(-h --help)'{-h,--help}'[Display usage]'
 	'*:packages:_devtools_completions_all_packages'

doc/man/diffpkg.1.asciidoc

@@ -21,7 +21,7 @@ When given one package, use it to diff against the locally built one.
 When given two packages, diff both packages against each other.
 
 In either case, a package name will be converted to a filename from the
-cache, and diffpkg will proceed as though this filename was initially
+cache or pool, and diffpkg will proceed as though this filename was initially
 specified.
 
 Options
@@ -30,12 +30,32 @@ Options
 *-M, --makepkg-config*::
 	Set an alternate makepkg configuration file
 
+
+*-P, --pool*='DIR'::
+	Search diff target in pool dir (default `'/srv/ftp/pool'`)
+
 *-v, --verbose*::
 	Provide more detailed/unfiltered output
 
 *-h, --help*::
 	Show a help text
 
+Output Options
+--------------
+
+*--color*[='WHEN']::
+	Color output; 'WHEN' is `'never'`, `'always'`, or `'auto'`; Plain *--color* means *--color='auto'*
+
+*-u, -U, --unified*::
+	Output 3 lines of unified context
+
+*-y, --side-by-side*::
+	Output in two columns
+
+*-W, --width*[='NUM']::
+	Output at most 'NUM' (default `'auto'`) print columns; 'NUM' can be `'auto'`, `'columns'` or a number.
+	`'auto'` will be resolved to the maximum line length of both files, guaranteeing the diff to be uncut.
+
 Modes
 -----
 

lib/common.sh

@@ -19,8 +19,6 @@ export LANG=C
 export BUILDTOOL=devtools
 export BUILDTOOLVER=m4_devtools_version
 
-shopt -s extglob
-
 # check if messages are to be printed using color
 if [[ -t 2 && "$TERM" != dumb ]]; then
 	colorize
@@ -142,12 +140,16 @@ pkgver_equal() {
 find_cached_package() {
 	local searchdirs=("$PWD" "$PKGDEST") results=()
 	local targetname=$1 targetver=$2 targetarch=$3
-	local dir pkg pkgbasename name ver rel arch r results
+	local dir pkg packages pkgbasename name ver rel arch r results
 
 	for dir in "${searchdirs[@]}"; do
 		[[ -d $dir ]] || continue
 
-		for pkg in "$dir"/*.pkg.tar?(.!(sig|*.*)); do
+		shopt -s extglob nullglob
+		mapfile -t packages < <(printf "%s\n" "$dir"/"${targetname}"-"${targetver}"-*"${targetarch}".pkg.tar?(.!(sig|*.*)))
+		shopt -u extglob nullglob
+
+		for pkg in "${packages[@]}"; do
 			[[ -f $pkg ]] || continue
 
 			# avoid adding duplicates of the same inode

src/checkpkg.in

@@ -2,8 +2,6 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-shopt -s extglob
-
 m4_include(lib/common.sh)
 
 usage() {
@@ -52,7 +50,7 @@ while (( $# )); do
 			shift
 			break
 			;;
-		-*,--*)
+		-*|--*)
 			die "invalid argument: %s" "$1"
 			;;
 		*)

src/crossrepomove.in

@@ -57,9 +57,14 @@ for _arch in "${arch[@]}"; do
 	fi
 	for _pkgname in "${pkgname[@]}"; do
 		fullver=$(get_full_version "$_pkgname")
-		pkgpath="/srv/ftp/$source_repo/os/$repo_arch/$_pkgname-$fullver-${_arch}.pkg.tar.*"
+		pkgpath="/srv/ftp/${source_repo}/os/${repo_arch}/${_pkgname}-${fullver}-${_arch}.pkg.tar.*"
+		debugpath="/srv/ftp/${source_repo}-debug/os/${repo_arch}/${_pkgname}-debug-${fullver}-${_arch}.pkg.tar.*"
+		# Fail if $pkgpath doesn't match but keep $debugpath optional
 		# shellcheck disable=2029
-		ssh "$server" "cp $pkgpath staging/$target_repo" || die
+		ssh "${server}" "bash -c '
+			install ${pkgpath} -Dt staging/${target_repo} &&
+			(install ${debugpath} -Dt staging/${target_repo} 2>/dev/null || true)
+		'" || die
 	done
 done
 

src/diffpkg.in

@@ -2,8 +2,6 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-shopt -s extglob
-
 m4_include(lib/common.sh)
 
 usage() {
@@ -24,9 +22,18 @@ usage() {
 
 		OPTIONS
 		    -M, --makepkg-config Set an alternate makepkg configuration file
+		    -P, --pool=DIR       Search diff target in pool dir (default '/srv/ftp/pool')
 		    -v, --verbose        Provide more detailed/unfiltered output
 		    -h, --help           Show this help text
 
+		OUTPUT OPTIONS
+		    --color[=WHEN]       Color output; WHEN is 'never', 'always', or 'auto';
+		                         Plain --color means --color='auto'
+		    -u, -U, --unified    Output 3 lines of unified context
+		    -y, --side-by-side   Output in two columns
+		    -W, --width=NUM      Output at most NUM (default 'auto') print columns
+		                         NUM can be 'auto', 'columns' or a number
+
 		MODES
 		    -l, --list           Activate content list diff mode (default)
 		    -d, --diffoscope     Activate diffoscope diff mode
@@ -36,12 +43,19 @@ _EOF_
 }
 
 MAKEPKG_CONF=/etc/makepkg.conf
+POOLDIR=/srv/ftp/pool
+
 VERBOSE=0
 TARLIST=0
 DIFFOSCOPE=0
 PKGINFO=0
 BUILDINFO=0
 
+DIFFMODE=--side-by-side
+DIFFCOLOR=--color=auto
+DIFFWIDTH=--width=auto
+DIFFOPTIONS=(--expand-tabs)
+
 # option checking
 while (( $# )); do
 	case $1 in
@@ -50,6 +64,7 @@ while (( $# )); do
 			exit 0
 			;;
 		-M|--makepkg-config)
+			(( $# <= 1 )) && die "missing argument for %s" "$1"
 			MAKEPKG_CONF="$2"
 			shift 2
 			;;
@@ -73,11 +88,50 @@ while (( $# )); do
 			VERBOSE=1
 			shift
 			;;
+		-u|-U|--unified)
+			DIFFMODE=--unified
+			shift
+			;;
+		-y|--side-by-side)
+			DIFFMODE=--side-by-side
+			shift
+			;;
+		--color|--color=*)
+			if [[ $2 == never || $2 == always || $2 == auto ]]; then
+				DIFFCOLOR="--color=$2"
+				shift 2
+				continue
+			fi
+			if [[ $1 == --color ]]; then
+				DIFFCOLOR="--color=auto"
+			else
+				DIFFCOLOR="$1"
+			fi
+			shift
+			;;
+		-W|--width)
+			(( $# <= 1 )) && die "missing argument for %s" "$1"
+			DIFFWIDTH="--width=$2"
+			shift 2
+			;;
+		--width=*)
+			DIFFWIDTH="$1"
+			shift
+			;;
+		-P|--pool)
+			(( $# <= 1 )) && die "missing argument for %s" "$1"
+			POOLDIR="$2"
+			shift 2
+			;;
+		--pool=*)
+			POOLDIR="${1#*=}"
+			shift
+			;;
 		--)
 			shift
 			break
 			;;
-		-*,--*)
+		-*|--*)
 			die "invalid argument: %s" "$1"
 			;;
 		*)
@@ -86,6 +140,22 @@ while (( $# )); do
 	esac
 done
 
+# Set options based on flags or magic values
+if (( VERBOSE )); then
+	if [[ $DIFFMODE == --unified ]]; then
+		DIFFMODE="--unified=99999"
+	fi
+else
+	DIFFOPTIONS+=(--suppress-common-lines)
+fi
+if [[ $DIFFWIDTH == --width=columns ]]; then
+	DIFFWIDTH="--width=${COLUMNS:-130}"
+fi
+if [[ $DIFFWIDTH != --width=auto ]]; then
+	DIFFOPTIONS+=("${DIFFWIDTH}")
+fi
+DIFFOPTIONS+=("${DIFFMODE}" "${DIFFCOLOR}")
+
 if ! (( DIFFOSCOPE || TARLIST || PKGINFO || BUILDINFO )); then
 	TARLIST=1
 fi
@@ -120,6 +190,19 @@ tar_list() {
 	fi | sort
 }
 
+file_line_length() {
+	path="$1"
+	wc -L "${path}" | tail -n1 | sed -E 's/^ +//g' | cut -d' ' -f1
+}
+
+file_diff_columns() {
+	file1="$1"
+	file2="$2"
+	file1_length=$(file_line_length "$file1")
+	file2_length=$(file_line_length "$file2")
+	echo $(( file1_length + file2_length + 3 ))
+}
+
 diff_pkgs() {
 	local oldpkg newpkg
 	oldpkg=$(readlink -m "$1")
@@ -128,29 +211,40 @@ diff_pkgs() {
 	[[ -f $oldpkg ]] || die "No such file: %s" "${oldpkg}"
 	[[ -f $newpkg ]] || die "No such file: %s" "${newpkg}"
 
-	if (( TARLIST )); then
-		tar_list "$oldpkg" > "$TMPDIR/filelist-old"
-		tar_list "$newpkg" > "$TMPDIR/filelist"
+	DIFFOPTIONS+=(--label "${oldpkg}" --label "${newpkg}")
 
-		sdiff -s "$TMPDIR/filelist-old" "$TMPDIR/filelist"
+	if (( TARLIST )); then
+		tar_list "$oldpkg" > "$TMPDIR/old"
+		tar_list "$newpkg" > "$TMPDIR/new"
 	fi
 
 	if (( PKGINFO )); then
-		bsdtar xOqf "$oldpkg" .PKGINFO > "$TMPDIR/pkginfo-old"
-		bsdtar xOqf "$newpkg" .PKGINFO > "$TMPDIR/pkginfo"
-
-		sdiff -s "$TMPDIR/pkginfo-old" "$TMPDIR/pkginfo"
+		bsdtar xOqf "$oldpkg" .PKGINFO > "$TMPDIR/old"
+		bsdtar xOqf "$newpkg" .PKGINFO > "$TMPDIR/new"
 	fi
 
 	if (( BUILDINFO )); then
-		bsdtar xOqf "$oldpkg" .BUILDINFO > "$TMPDIR/buildinfo-old"
-		bsdtar xOqf "$newpkg" .BUILDINFO > "$TMPDIR/buildinfo"
+		bsdtar xOqf "$oldpkg" .BUILDINFO > "$TMPDIR/old"
+		bsdtar xOqf "$newpkg" .BUILDINFO > "$TMPDIR/new"
+	fi
 
-		sdiff -s "$TMPDIR/buildinfo-old" "$TMPDIR/buildinfo"
+	if (( TARLIST || PKGINFO || BUILDINFO )); then
+		# Resolve dynamic auto width one we know the content to diff
+		if [[ $DIFFWIDTH == --width=auto ]]; then
+			AUTOLENGTH=$(file_diff_columns "$TMPDIR/old" "$TMPDIR/new")
+			DIFFOPTIONS+=("--width=${AUTOLENGTH}")
+		fi
+
+		# Print a header for side-by-side view as it lacks labels
+		if [[ $DIFFMODE == --side-by-side ]]; then
+			printf -- "--- %s\n+++ %s\n" "${oldpkg}" "${newpkg}"
+		fi
+
+		diff "${DIFFOPTIONS[@]}" "$TMPDIR/old" "$TMPDIR/new"
 	fi
 
 	if (( DIFFOSCOPE )); then
-		diffoscope "$oldpkg" "$newpkg"
+		diffoscope "${DIFFCOLOR/--color/--text-color}" "$oldpkg" "$newpkg"
 	fi
 }
 
@@ -169,8 +263,19 @@ fetch_pkg() {
 			pkg=$1 ;;
 	esac
 
-	[[ -n $pkgurl ]] || pkgurl=$(pacman -Spdd --print-format '%l' --noconfirm "$pkg") ||
-		die "Couldn't download previous package for %s." "$pkg"
+	if [[ -z ${pkgurl} ]]; then
+		# Try to find latest package in pool dir
+		if [[ -d ${POOLDIR} ]]; then
+			shopt -s extglob nullglob
+			pkgurl=$(printf "%s\n" "${POOLDIR}"/*/"${_pkgname}"-!(*-*)-!(*-*)-!(*-*).pkg.tar!(*.sig)|sort -Vr|head -1)
+			shopt -u extglob nullglob
+		fi
+		# Search via pacman database if no pool file exists
+		if [[ ! -f ${pkgurl} ]]; then
+			pkgurl=$(pacman -Spdd --print-format '%l' --noconfirm "$pkg") ||
+				die "Couldn't download previous package for %s." "$pkg"
+		fi
+	fi
 
 	pkg=${pkgurl##*/}
 	pkgdest=$(mktemp -t -d "${pkg}-XXXXXX")/${pkg}

src/find-libdeps.in

@@ -5,7 +5,6 @@
 m4_include(lib/common.sh)
 
 set -e
-shopt -s extglob
 
 IGNORE_INTERNAL=0
 
@@ -43,7 +42,9 @@ fi
 
 process_sofile() {
 	# extract the library name: libfoo.so
+	shopt -s extglob nullglob
 	soname="${sofile%.so?(+(.+([0-9])))}".so
+	shopt -u extglob nullglob
 	# extract the major version: 1
 	soversion="${sofile##*\.so\.}"
 	if [[ "$soversion" = "$sofile" ]] && ((IGNORE_INTERNAL)); then

src/makechrootpkg.in

@@ -311,13 +311,14 @@ fi
 # Pass all arguments after -- right to makepkg
 makepkg_args+=("${@:$OPTIND}")
 
-# See if -R or -e was passed to makepkg
+# See if -R, -e or -A was passed to makepkg
 for arg in "${@:$OPTIND}"; do
 	case ${arg%%=*} in
-		--skip*|--holdver) verifysource_args+=("$arg") ;;
+		--skip*|--holdver|--ignorearch) verifysource_args+=("$arg") ;;
 		--repackage|--noextract) keepbuilddir=1 ;;
 		--*) ;;
-		-*R*|-*e*) keepbuilddir=1 ;;
+		-*A*) verifysource_args+=(-A) ;;&
+		-*R*|-*e*) keepbuilddir=1 ;;&
 	esac
 done
 

src/makerepropkg.in

@@ -86,13 +86,14 @@ get_pkgfile() {
 
 get_makepkg_conf() {
     local fname=${1}
-    local makepkg_conf="${2}"
+    local arch="${2}"
+    local makepkg_conf="${3}"
     if ! buildtool_file=$(get_pkgfile "${fname}"); then
         error "failed to retrieve ${fname}"
         return 1
     fi
     msg2 "using makepkg.conf from ${fname}"
-    bsdtar xOqf "${buildtool_file/file:\/\//}" usr/share/devtools/makepkg-x86_64.conf > "${makepkg_conf}"
+    bsdtar xOqf "${buildtool_file/file:\/\//}" "usr/share/devtools/makepkg-${arch}.conf" > "${makepkg_conf}"
     return 0
 }
 
@@ -213,17 +214,17 @@ TEMPDIR=$(mktemp -d --tmpdir makerepropkg.XXXXXXXXXX)
 makepkg_conf="${TEMPDIR}/makepkg.conf"
 # anything before buildtool support is pinned to the last none buildtool aware release
 if [[ -z "${BUILDTOOL}" ]]; then
-    get_makepkg_conf "devtools-20210202-3-any" "${makepkg_conf}" || exit 1
+    get_makepkg_conf "devtools-20210202-3-any" "${CARCH}" "${makepkg_conf}" || exit 1
 # prefere to assume devtools up until matching makepkg version so repository packages remain reproducible
 elif [[ "${BUILDTOOL}" = makepkg ]] && (( $(vercmp "${BUILDTOOLVER}" 6.0.1) <= 0 )); then
-    get_makepkg_conf "devtools-20210202-3-any" "${makepkg_conf}" || exit 1
+    get_makepkg_conf "devtools-20210202-3-any" "${CARCH}" "${makepkg_conf}" || exit 1
 # all devtools builds
-elif [[ "${BUILDTOOL}" = devtools ]] && get_makepkg_conf "${BUILDTOOL}-${BUILDTOOLVER}" "${makepkg_conf}"; then
+elif [[ "${BUILDTOOL}" = devtools ]] && get_makepkg_conf "${BUILDTOOL}-${BUILDTOOLVER}" "${CARCH}" "${makepkg_conf}"; then
     true
 # fallback to current makepkg.conf
 else
     warning "Unknown buildtool (${BUILDTOOL}-${BUILDTOOLVER}), using fallback"
-    makepkg_conf=@pkgdatadir@/makepkg-x86_64.conf
+    makepkg_conf=@pkgdatadir@/makepkg-${CARCH}.conf
 fi
 printf '%s\n' "${allpkgfiles[@]}" | mkarchroot -M "${makepkg_conf}" -U "${archroot_args[@]}" "${namespace}/root" - || exit 1
 

src/mkarchroot.in

@@ -81,7 +81,7 @@ for file in "${files[@]}"; do
 	cp "$file" "$working_dir$file"
 done
 
-unshare --mount pacstrap -${umode}Mcd ${pac_conf:+-C "$pac_conf"} "$working_dir" \
+unshare --mount pacstrap -${umode}Mc ${pac_conf:+-C "$pac_conf"} "$working_dir" \
 	"${cache_dirs[@]/#/--cachedir=}" "$@" || die 'Failed to install all packages'
 
 printf '%s.UTF-8 UTF-8\n' C en_US de_DE > "$working_dir/etc/locale.gen"