wip: clean all existing makepkg conf.d files before overriding

Previously we have only copied the passed makepkg.conf file into the
chroot, which misses build flags for additional language specific files
that makepkg supports. Fix this by extracting all conf.d makepkg config
files from the detected devtools archive.

Component: makerepropkg
Co-authored-by: Christian Heusel <christian@heusel.eu>

Makefile

@@ -1,6 +1,6 @@
 SHELL=/bin/bash -o pipefail
 
-V=1.3.0
+V=1.3.1
 BUILDTOOLVER ?= $(V)
 
 PREFIX = /usr/local
@@ -106,7 +106,7 @@ $(BUILDDIR)/doc/man/%: doc/man/%.asciidoc doc/man/include/footer.asciidoc
 
 conf:
 	@install -d $(BUILDDIR)/makepkg.conf.d
-	@cp -a $(MAKEPKG_CONFIGS) $(BUILDDIR)/makepkg.conf.d
+	@cp -ra $(MAKEPKG_CONFIGS) $(BUILDDIR)/makepkg.conf.d
 	@install -d $(BUILDDIR)/pacman.conf.d
 	@cp -a $(PACMAN_CONFIGS) $(BUILDDIR)/pacman.conf.d
 	@install -d $(BUILDDIR)/git.conf.d
@@ -124,7 +124,7 @@ install: all
 	install -dm0755 $(DESTDIR)$(DATADIR)/lib
 	cp -ra $(BUILDDIR)/lib/* $(DESTDIR)$(DATADIR)/lib
 	cp -a $(BUILDDIR)/git.conf.d -t $(DESTDIR)$(DATADIR)
-	for conf in $(notdir $(MAKEPKG_CONFIGS)); do install -Dm0644 $(BUILDDIR)/makepkg.conf.d/$$conf $(DESTDIR)$(DATADIR)/makepkg.conf.d/$${conf##*/}; done
+	cp -ra $(BUILDDIR)/makepkg.conf.d -t $(DESTDIR)$(DATADIR)
 	for conf in $(notdir $(PACMAN_CONFIGS)); do install -Dm0644 $(BUILDDIR)/pacman.conf.d/$$conf $(DESTDIR)$(DATADIR)/pacman.conf.d/$${conf##*/}; done
 	for a in ${SETARCH_ALIASES}; do install -m0644 $$a -t $(DESTDIR)$(DATADIR)/setarch-aliases.d; done
 	for l in ${COMMITPKG_LINKS}; do ln -sf commitpkg $(DESTDIR)$(PREFIX)/bin/$$l; done
@@ -142,7 +142,7 @@ uninstall:
 	for f in $(notdir $(LIBRARY)); do rm -f $(DESTDIR)$(DATADIR)/lib/$$f; done
 	rm -rf $(DESTDIR)$(DATADIR)/lib
 	rm -rf $(DESTDIR)$(DATADIR)/git.conf.d
-	for conf in $(notdir $(MAKEPKG_CONFIGS)); do rm -f $(DESTDIR)$(DATADIR)/makepkg.conf.d/$${conf##*/}; done
+	rm -rf $(DESTDIR)$(DATADIR)/makepkg.conf.d
 	for conf in $(notdir $(PACMAN_CONFIGS)); do rm -f $(DESTDIR)$(DATADIR)/pacman.conf.d/$${conf##*/}; done
 	for f in $(notdir $(SETARCH_ALIASES)); do rm -f $(DESTDIR)$(DATADIR)/setarch-aliases.d/$$f; done
 	for l in ${COMMITPKG_LINKS}; do rm -f $(DESTDIR)$(PREFIX)/bin/$$l; done
@@ -154,7 +154,6 @@ uninstall:
 	for manfile in $(notdir $(MANS)); do rm -f $(DESTDIR)$(MANDIR)/man$${manfile##*.}/$${manfile}; done;
 	rmdir --ignore-fail-on-non-empty \
 		$(DESTDIR)$(DATADIR)/setarch-aliases.d \
-		$(DESTDIR)$(DATADIR)/makepkg.conf.d \
 		$(DESTDIR)$(DATADIR)/pacman.conf.d \
 		$(DESTDIR)$(DATADIR)
 

config/makepkg/conf.d/fortran.conf

@@ -0,0 +1,18 @@
+#!/hint/bash
+#
+# /etc/makepkg.conf.d/fortran.conf
+#
+
+#########################################################################
+# FORTRAN LANGUAGE SUPPORT
+#########################################################################
+
+# Flags used for the Fortran compiler, similar in spirit to CFLAGS. Read
+# linkman:gfortran[1] for more details on the available flags.
+#FFLAGS="-O2 -pipe"
+#FCFLAGS="$FFLAGS"
+
+# Additional compiler flags appended to `FFLAGS` and `FCFLAGS` for use in debugging. Usually
+# this would include: ``-g''. Read linkman:gfortran[1] for more details on the wide
+# variety of compiler flags available.
+#DEBUG_FFLAGS="-g"

config/makepkg/conf.d/rust.conf

@@ -0,0 +1,19 @@
+#!/hint/bash
+# shellcheck disable=2034
+
+#
+# /etc/makepkg.conf.d/rust.conf
+#
+
+#########################################################################
+# RUST LANGUAGE SUPPORT
+#########################################################################
+
+# Flags used for the Rust compiler, similar in spirit to CFLAGS. Read
+# linkman:rustc[1] for more details on the available flags.
+RUSTFLAGS="-Cforce-frame-pointers=yes"
+
+# Additional compiler flags appended to `RUSTFLAGS` for use in debugging.
+# Usually this would include: ``-C debuginfo=2''. Read linkman:rustc[1] for
+# more details on the available flags.
+DEBUG_RUSTFLAGS="-C debuginfo=2"

config/makepkg/x86_64.conf.d/fortran.conf

@@ -0,0 +1 @@
+../conf.d/fortran.conf

config/makepkg/x86_64.conf.d/rust.conf

@@ -0,0 +1 @@
+../conf.d/rust.conf

config/makepkg/x86_64_v3.conf.d/fortran.conf

@@ -0,0 +1 @@
+../conf.d/fortran.conf

config/makepkg/x86_64_v3.conf.d/rust.conf

@@ -0,0 +1 @@
+../conf.d/rust.conf

contrib/completion/zsh/_devtools.in

@@ -202,7 +202,7 @@ _pkgctl_issue_view_args=(
 
 _pkgctl_release_args=(
 	'(-m --message=)'{-m,--message=}"[Use the given <msg> as the commit message]:message:"
-	'(-r --repo=)'{-r,--repo=}"[Specify a target repository for new packages]:repo:($DEVTOOLS_VALID_REPOS[*])"
+	'(-r --repo)'{-r,--repo}"[Specify a target repository for new packages]:repo:($DEVTOOLS_VALID_REPOS[*])"
 	'(-s --staging)'{-s,--staging}'[Release to the staging counterpart of the auto-detected repo]'
 	'(-t --testing)'{-t,--testing}'[Release to the testing counterpart of the auto-detected repo]'
 	'(-u --db-update)'{-u,--db-update}'[Automatically update the pacman database after uploading]'

doc/man/arch-nspawn.1.asciidoc

@@ -23,7 +23,8 @@ Options
 	Location of a pacman config file
 
 *-M* <file>::
-	Location of a makepkg config file
+	Location of a makepkg config file. Specific additions (e.g. build flags for
+	additional languages) can be placed in '<file>.d/*.conf'.
 
 *-c* <dir>::
 	Set pacman cache, if no directory is specified the passed pacman.conf's cachedir is used with a fallback to '/etc/pacman.conf'

doc/man/makerepropkg.1.asciidoc

@@ -49,7 +49,8 @@ Options
 	Set the pacman cache directory.
 
 *-M* <file>::
-	Location of a makepkg config file.
+	Location of a makepkg config file. Specific additions (e.g. build flags for
+	additional languages) can be placed in '<file>.d/*.conf'.
 
 *-l* <chroot>::
 	The directory name to use as the chroot namespace

src/arch-nspawn.in

@@ -112,7 +112,16 @@ copy_hostconf () {
 	[[ -n $host_cachemirrors ]] && printf 'CacheServer = %s\n' "${host_cachemirrors[@]}" >>"$working_dir/etc/pacman.d/mirrorlist"
 
 	[[ -n $pac_conf ]] && cp "$pac_conf" "$working_dir/etc/pacman.conf"
-	[[ -n $makepkg_conf ]] && cp "$makepkg_conf" "$working_dir/etc/makepkg.conf"
+	if [[ -n $makepkg_conf ]]; then
+		cp "$makepkg_conf" "$working_dir/etc/makepkg.conf"
+		if [[ -d "${makepkg_conf}.d" ]] && is_globfile "${makepkg_conf}.d"/*.conf; then
+			mkdir --parents "$working_dir/etc/makepkg.conf.d/"
+			if is_globfile "$working_dir/etc/makepkg.conf.d"/*.conf; then
+				rm --force "$working_dir/etc/makepkg.conf.d"/*.conf
+			fi
+			cp "${makepkg_conf}.d/"*.conf "$working_dir/etc/makepkg.conf.d/"
+		fi
+	fi
 
 	local file
 	for file in "${files[@]}"; do

src/archbuild.in

@@ -79,6 +79,13 @@ check_root SOURCE_DATE_EPOCH,SRCDEST,SRCPKGDEST,PKGDEST,LOGDEST,MAKEFLAGS,PACKAG
 # Pass all arguments after -- right to makepkg
 makechrootpkg_args+=("${@:$OPTIND}")
 
+# Automatically recreate the root chroot if a version mismatch is detected
+CURRENT_CHROOT_VERSION=$(cat "${chroots}/${repo}-${arch}/root/.arch-chroot")
+if [[ -f "${chroots}/${repo}-${arch}/root/.arch-chroot" ]] && [[ "$CURRENT_CHROOT_VERSION" != "$CHROOT_VERSION" ]]; then
+	warning "Recreating chroot '%s' (%s) as it is not at version %s" "${chroots}/${repo}-${arch}/root" "$CURRENT_CHROOT_VERSION" "$CHROOT_VERSION"
+	clean_first=true
+fi
+
 if ${clean_first} || [[ ! -d "${chroots}/${repo}-${arch}" ]]; then
 	msg "Creating chroot for [%s] (%s)..." "${repo}" "${arch}"
 

src/checkpkg.in

@@ -140,7 +140,7 @@ for _pkgname in "${pkgname[@]}"; do
 	bsdtar tf "$TEMPDIR/$oldpkg" | sort > "$TEMPDIR/filelist-$_pkgname-old"
 	bsdtar tf "$pkgfile" | sort > "$TEMPDIR/filelist-$_pkgname"
 
-	diff --side-by-side --suppress-common-lines --width="$COLUMNS" --color=auto "$TEMPDIR/filelist-$_pkgname-old" "$TEMPDIR/filelist-$_pkgname"
+	diff --side-by-side --suppress-common-lines --width="${COLUMNS:-130}" --color=auto "$TEMPDIR/filelist-$_pkgname-old" "$TEMPDIR/filelist-$_pkgname"
 
 	find-libprovides "$TEMPDIR/$oldpkg" 2>/dev/null | sort > "$TEMPDIR/libraries-$_pkgname-old"
 	find-libprovides "$pkgfile" 2>/dev/null | sort > "$TEMPDIR/libraries-$_pkgname"

src/lib/api/archweb.sh

@@ -14,6 +14,8 @@ set -o pipefail
 
 
 archweb_query_all_packages() {
+	local -a pkgbases
+
 	[[ -z ${WORKDIR:-} ]] && setup_workdir
 
 	stat_busy "Query all released packages"
@@ -36,6 +38,7 @@ archweb_query_all_packages() {
 
 archweb_query_maintainer_packages() {
 	local maintainer=$1
+	local -a pkgbases
 
 	[[ -z ${WORKDIR:-} ]] && setup_workdir
 

src/lib/api/gitlab.sh

@@ -119,11 +119,13 @@ gitlab_api_call_paged() {
 
 	local next_page=1
 	local total_pages=1
+	local known_total_pages=1
+	local percentage=100
 
 	while [[ -n "${next_page}" ]]; do
 		percentage=$(( 100 * next_page / total_pages ))
 		printf "📡 Querying GitLab: %s/%s [%s] %%spinner%%" \
-			"${BOLD}${next_page}" "${total_pages}" "${percentage}%${ALL_OFF}"  \
+			"${BOLD}${next_page}" "${known_total_pages}" "${percentage}%${ALL_OFF}"  \
 			> "${tmp_file}"
 		mv "${tmp_file}" "${status_file}"
 
@@ -148,6 +150,15 @@ gitlab_api_call_paged() {
 
 		next_page=$(grep "x-next-page" "${header}" | tr -d '\r' | awk '{ print $2 }')
 		total_pages=$(grep "x-total-pages" "${header}" | tr -d '\r' | awk '{ print $2 }')
+		# The api is not guaranteed to return x-total-pages for larger query results
+		# https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/23931
+		# https://gitlab.com/gitlab-org/gitlab/-/issues/436373
+		if (( total_pages == 0 )); then
+			total_pages=${next_page}
+			known_total_pages="?"
+		else
+			known_total_pages=${total_pages}
+		fi
 	done
 
 	jq --slurp add "${api_workdir}"/result.* > "${outfile}"

src/lib/archroot.sh

@@ -4,7 +4,7 @@
 :
 
 # shellcheck disable=2034
-CHROOT_VERSION='v5'
+CHROOT_VERSION='v6'
 
 ##
 #  usage : check_root $keepenv

src/lib/issue/view.sh

@@ -181,7 +181,7 @@ pkgctl_issue_view() {
 		status_dir=$(mktemp --tmpdir="${WORKDIR}" --directory pkgctl-gitlab-api.XXXXXXXXXX)
 		printf "📡 Querying GitLab issue notes API..." > "${status_dir}/status"
 		term_spinner_start "${status_dir}"
-		if ! output=$(gitlab_project_issue_notes "${project_path}" "${iid}" "${status_dir}/status"); then
+		if ! output=$(gitlab_project_issue_notes "${project_path}" "${iid}" "${status_dir}/status" "sort=asc&order_by=created_at"); then
 			term_spinner_stop "${status_dir}"
 			msg_error "Failed to fetch comments"
 			exit 1

src/lib/repo/clone.sh

@@ -65,6 +65,7 @@ pkgctl_repo_clone() {
 	local CONFIGURE_OPTIONS=()
 	local jobs=
 	jobs=$(nproc)
+	local -a pkgbases
 
 	# variables
 	local command=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}}

src/lib/util/pacman.sh

@@ -33,6 +33,7 @@ update_pacman_repo_cache() {
 get_pacman_repo_from_pkgbuild() {
 	local path=${1:-PKGBUILD}
 	local repo=${2:-multilib}
+	local -a pkgnames
 
 	# shellcheck source=contrib/makepkg/PKGBUILD.proto
 	mapfile -t pkgnames < <(source "${path}"; printf "%s\n" "${pkgname[@]}")
@@ -67,6 +68,7 @@ get_pkgnames_from_repo_pkgbase() {
 	local repo=$1
 	shift
 	local pkgbases=("$@")
+	local -a pkgnames
 
 	# update the pacman repo cache if it doesn't exist yet
 	if [[ ! -d "${_DEVTOOLS_PACMAN_CACHE_DIR}" ]]; then

src/lib/version/check.sh

@@ -292,7 +292,8 @@ get_upstream_version() {
 		opts+=(--keyfile "${keyfile}")
 	fi
 
-	if ! output=$(GIT_TERMINAL_PROMPT=0 nvchecker --file "${config}" --logger json "${opts[@]}" 2>&1 | \
+	if ! output=$(GIT_TERMINAL_PROMPT=0 GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null \
+			nvchecker --file "${config}" --logger json "${opts[@]}" 2>&1 | \
 			jq --raw-output 'select((.level != "debug") and (.event != "ignoring invalid version"))'); then
 		printf "failed to run nvchecker: %s" "${output}"
 		return 1

src/makerepropkg.in

@@ -93,14 +93,36 @@ get_makepkg_conf() {
     local fname=${1}
     local arch="${2}"
     local makepkg_conf="${3}"
+
     if ! buildtool_file=$(get_pkgfile "${fname}"); then
-        error "failed to retrieve ${fname}"
+       error "failed to retrieve ${fname}"
+       return 1
+    fi
+    buildtool_file="${buildtool_file/file:\/\//}"
+    msg "using makepkg.conf from ${fname}"
+
+    # try to handle config of legacy devtools
+    if bsdtar --list --file "${buildtool_file}" "usr/share/devtools/makepkg-${arch}.conf" &>/dev/null; then
+        bsdtar --extract --to-stdout --fast-read --file "${buildtool_file}" "usr/share/devtools/makepkg-${arch}.conf" > "${makepkg_conf}"
+        return $?
+    fi
+
+    msg2 "extracting ${arch}.conf from devtools archive"
+    if ! bsdtar --extract --to-stdout --fast-read --file "${buildtool_file}" "usr/share/devtools/makepkg.conf.d/${arch}.conf" > "${makepkg_conf}"; then
+        error "failed to extract 'usr/share/devtools/makepkg.conf.d/${arch}.conf' from devtools archive"
         return 1
     fi
-    msg2 "using makepkg.conf from ${fname}"
+
-	if ! bsdtar xOqf "${buildtool_file/file:\/\//}" "usr/share/devtools/makepkg.conf.d/${arch}.conf" > "${makepkg_conf}"; then
+    mkdir --parents "${makepkg_conf}.d"
-		bsdtar xOqf "${buildtool_file/file:\/\//}" "usr/share/devtools/makepkg-${arch}.conf" > "${makepkg_conf}"
+    if bsdtar --list --file "${buildtool_file}" "usr/share/devtools/makepkg.conf.d/conf.d" &>/dev/null; then
-	fi
+        msg2 "extracting conf.d from devtools archive"
+        bsdtar --extract --file "${buildtool_file}" --cd "${makepkg_conf}.d" --strip-components 4 "usr/share/devtools/makepkg.conf.d/conf.d"
+    fi
+    if bsdtar --list --file "${buildtool_file}" "usr/share/devtools/makepkg.conf.d/${arch}.conf.d" &>/dev/null; then
+        msg2 "extracting ${arch}.conf.d from devtools archive"
+        bsdtar --extract --file "${buildtool_file}" --cd "${makepkg_conf}.d" --strip-components 4 "usr/share/devtools/makepkg.conf.d/${arch}.conf.d"
+    fi
+
     return 0
 }
 
@@ -186,7 +208,7 @@ for f in "${splitpkgs[@]}"; do
 done
 
 if (( ${#cache_dirs[@]} == 0 )); then
-	mapfile -t cache_dirs < <(pacman-conf CacheDir)
+    mapfile -t cache_dirs < <(pacman-conf CacheDir)
 fi
 
 ORIG_HOME=${HOME}

src/offload-build.in

@@ -114,25 +114,32 @@ ssh "${SSH_OPTS[@]}" -- "$server" "cd ${remote_temp@Q} && bsdtar --strip-compone
 # Run the build command on the server
 msg "Running archbuild"
 # shellcheck disable=SC2145
-ssh "${SSH_OPTS[@]}" -t -- "$server" "cd ${remote_temp@Q} && export LOGDEST="" && ${archbuild_cmd[@]@Q}" || die
+if ssh "${SSH_OPTS[@]}" -t -- "$server" "cd ${remote_temp@Q} && export LOGDEST="" && ${archbuild_cmd[@]@Q}"; then
-msg "Build complete"
+    msg "Build complete"
 
-# Get an array of files that should be downloaded from the server
+    # Get an array of files that should be downloaded from the server
-mapfile -t files < <(
+    mapfile -t files < <(
-    ssh "${SSH_OPTS[@]}" -- "$server" "
+        ssh "${SSH_OPTS[@]}" -- "$server" "
-        cd ${remote_temp@Q}"' &&
+            cd ${remote_temp@Q}"' &&
-        makepkg_user_config="${XDG_CONFIG_HOME:-$HOME/.config}/pacman/makepkg.conf" &&
+            makepkg_user_config="${XDG_CONFIG_HOME:-$HOME/.config}/pacman/makepkg.conf" &&
-        makepkg_config="/usr/share/devtools/makepkg.conf.d/'"${arch}"'.conf" &&
+            makepkg_config="/usr/share/devtools/makepkg.conf.d/'"${arch}"'.conf" &&
-        if [[ -f /usr/share/devtools/makepkg.conf.d/'"${repo}"'-'"${arch}"'.conf ]]; then
+            if [[ -f /usr/share/devtools/makepkg.conf.d/'"${repo}"'-'"${arch}"'.conf ]]; then
-            makepkg_config="/usr/share/devtools/makepkg.conf.d/'"${repo}"'-'"${arch}"'.conf"
+                makepkg_config="/usr/share/devtools/makepkg.conf.d/'"${repo}"'-'"${arch}"'.conf"
-        fi &&
+            fi &&
-        while read -r file; do
+            while read -r file; do
-            [[ -f "${file}" ]] && printf "%s\n" "${file}" ||:
+                [[ -f "${file}" ]] && printf "%s\n" "${file}" ||:
-        done < <(makepkg --config <(cat "${makepkg_user_config}" "${makepkg_config}" 2>/dev/null) --packagelist) &&
+            done < <(makepkg --config <(cat "${makepkg_user_config}" "${makepkg_config}" 2>/dev/null) --packagelist) &&
-        printf "%s\n" '"${remote_temp@Q}/PKGBUILD"'
+            printf "%s\n" '"${remote_temp@Q}/PKGBUILD"'
 
-        find '"${remote_temp@Q}"' -name "*.log"
+            find '"${remote_temp@Q}"' -name "*.log"
-')
+    ')
+else
+    # Build failed, only the logs should be downloaded from the server
+    mapfile -t files < <(
+        ssh "${SSH_OPTS[@]}" -- "$server" '
+            find '"${remote_temp@Q}"' -name "*.log"
+    ')
+fi
 
 
 if (( ${#files[@]} )); then
@@ -142,13 +149,14 @@ if (( ${#files[@]} )); then
     if is_globfile "${TEMPDIR}"/*.log; then
         mv "${TEMPDIR}"/*.log "${LOGDEST:-${PWD}}/"
     fi
-    # missing PKGBUILD download means the build failed
+    if is_globfile "${TEMPDIR}"/*.pkg.tar*; then
-    if [[ ! -f "${TEMPDIR}/PKGBUILD" ]]; then
+        # Building a package may change the PKGBUILD during update_pkgver
+        mv "${TEMPDIR}/PKGBUILD" "${PWD}/"
+        mv "${TEMPDIR}"/*.pkg.tar* "${PKGDEST:-${PWD}}/"
+    else
         error "Build failed, check logs in ${LOGDEST:-${PWD}}"
         exit 1
     fi
-    mv "${TEMPDIR}/PKGBUILD" "${PWD}/"
-    mv "${TEMPDIR}"/*.pkg.tar* "${PKGDEST:-${PWD}}/"
 else
     exit 1
 fi