wip: test: devel makepkg package

Vendor all language related makepkg.conf files which are also shipped by
makepkg itself. This makes sure we always have full control over the
build flags inside devtools and overlay any by the vendored config we
maintain in devtools.

Component: archbuild
Signed-off-by: Levente Polyak <anthraxx@archlinux.org>

Makefile

@@ -1,6 +1,6 @@
 SHELL=/bin/bash -o pipefail
 
-V=1.3.0
+V=1.3.1
 BUILDTOOLVER ?= $(V)
 
 PREFIX = /usr/local
@@ -106,7 +106,7 @@ $(BUILDDIR)/doc/man/%: doc/man/%.asciidoc doc/man/include/footer.asciidoc
 
 conf:
 	@install -d $(BUILDDIR)/makepkg.conf.d
-	@cp -a $(MAKEPKG_CONFIGS) $(BUILDDIR)/makepkg.conf.d
+	@cp -ra $(MAKEPKG_CONFIGS) $(BUILDDIR)/makepkg.conf.d
 	@install -d $(BUILDDIR)/pacman.conf.d
 	@cp -a $(PACMAN_CONFIGS) $(BUILDDIR)/pacman.conf.d
 	@install -d $(BUILDDIR)/git.conf.d
@@ -124,7 +124,7 @@ install: all
 	install -dm0755 $(DESTDIR)$(DATADIR)/lib
 	cp -ra $(BUILDDIR)/lib/* $(DESTDIR)$(DATADIR)/lib
 	cp -a $(BUILDDIR)/git.conf.d -t $(DESTDIR)$(DATADIR)
-	for conf in $(notdir $(MAKEPKG_CONFIGS)); do install -Dm0644 $(BUILDDIR)/makepkg.conf.d/$$conf $(DESTDIR)$(DATADIR)/makepkg.conf.d/$${conf##*/}; done
+	cp -ra $(BUILDDIR)/makepkg.conf.d -t $(DESTDIR)$(DATADIR)
 	for conf in $(notdir $(PACMAN_CONFIGS)); do install -Dm0644 $(BUILDDIR)/pacman.conf.d/$$conf $(DESTDIR)$(DATADIR)/pacman.conf.d/$${conf##*/}; done
 	for a in ${SETARCH_ALIASES}; do install -m0644 $$a -t $(DESTDIR)$(DATADIR)/setarch-aliases.d; done
 	for l in ${COMMITPKG_LINKS}; do ln -sf commitpkg $(DESTDIR)$(PREFIX)/bin/$$l; done
@@ -142,7 +142,7 @@ uninstall:
 	for f in $(notdir $(LIBRARY)); do rm -f $(DESTDIR)$(DATADIR)/lib/$$f; done
 	rm -rf $(DESTDIR)$(DATADIR)/lib
 	rm -rf $(DESTDIR)$(DATADIR)/git.conf.d
-	for conf in $(notdir $(MAKEPKG_CONFIGS)); do rm -f $(DESTDIR)$(DATADIR)/makepkg.conf.d/$${conf##*/}; done
+	rm -rf $(DESTDIR)$(DATADIR)/makepkg.conf.d
 	for conf in $(notdir $(PACMAN_CONFIGS)); do rm -f $(DESTDIR)$(DATADIR)/pacman.conf.d/$${conf##*/}; done
 	for f in $(notdir $(SETARCH_ALIASES)); do rm -f $(DESTDIR)$(DATADIR)/setarch-aliases.d/$$f; done
 	for l in ${COMMITPKG_LINKS}; do rm -f $(DESTDIR)$(PREFIX)/bin/$$l; done
@@ -154,7 +154,6 @@ uninstall:
 	for manfile in $(notdir $(MANS)); do rm -f $(DESTDIR)$(MANDIR)/man$${manfile##*.}/$${manfile}; done;
 	rmdir --ignore-fail-on-non-empty \
 		$(DESTDIR)$(DATADIR)/setarch-aliases.d \
-		$(DESTDIR)$(DATADIR)/makepkg.conf.d \
 		$(DESTDIR)$(DATADIR)/pacman.conf.d \
 		$(DESTDIR)$(DATADIR)
 

config/makepkg/conf.d/fortran.conf

@@ -0,0 +1,18 @@
+#!/hint/bash
+#
+# /etc/makepkg.conf.d/fortran.conf
+#
+
+#########################################################################
+# FORTRAN LANGUAGE SUPPORT
+#########################################################################
+
+# Flags used for the Fortran compiler, similar in spirit to CFLAGS. Read
+# linkman:gfortran[1] for more details on the available flags.
+#FFLAGS="-O2 -pipe"
+#FCFLAGS="$FFLAGS"
+
+# Additional compiler flags appended to `FFLAGS` and `FCFLAGS` for use in debugging. Usually
+# this would include: ``-g''. Read linkman:gfortran[1] for more details on the wide
+# variety of compiler flags available.
+#DEBUG_FFLAGS="-g"

config/makepkg/conf.d/rust.conf

@@ -0,0 +1,19 @@
+#!/hint/bash
+# shellcheck disable=2034
+
+#
+# /etc/makepkg.conf.d/rust.conf
+#
+
+#########################################################################
+# RUST LANGUAGE SUPPORT
+#########################################################################
+
+# Flags used for the Rust compiler, similar in spirit to CFLAGS. Read
+# linkman:rustc[1] for more details on the available flags.
+RUSTFLAGS="-Cforce-frame-pointers=yes"
+
+# Additional compiler flags appended to `RUSTFLAGS` for use in debugging.
+# Usually this would include: ``-C debuginfo=2''. Read linkman:rustc[1] for
+# more details on the available flags.
+DEBUG_RUSTFLAGS="-C debuginfo=2"

config/makepkg/x86_64.conf.d/fortran.conf

@@ -0,0 +1 @@
+../conf.d/fortran.conf

config/makepkg/x86_64.conf.d/rust.conf

@@ -0,0 +1 @@
+../conf.d/rust.conf

config/makepkg/x86_64_v3.conf.d/fortran.conf

@@ -0,0 +1 @@
+../conf.d/fortran.conf

config/makepkg/x86_64_v3.conf.d/rust.conf

@@ -0,0 +1 @@
+../conf.d/rust.conf

contrib/completion/zsh/_devtools.in

@@ -202,7 +202,7 @@ _pkgctl_issue_view_args=(
 
 _pkgctl_release_args=(
 	'(-m --message=)'{-m,--message=}"[Use the given <msg> as the commit message]:message:"
-	'(-r --repo=)'{-r,--repo=}"[Specify a target repository for new packages]:repo:($DEVTOOLS_VALID_REPOS[*])"
+	'(-r --repo)'{-r,--repo}"[Specify a target repository for new packages]:repo:($DEVTOOLS_VALID_REPOS[*])"
 	'(-s --staging)'{-s,--staging}'[Release to the staging counterpart of the auto-detected repo]'
 	'(-t --testing)'{-t,--testing}'[Release to the testing counterpart of the auto-detected repo]'
 	'(-u --db-update)'{-u,--db-update}'[Automatically update the pacman database after uploading]'

src/archbuild.in

@@ -79,6 +79,13 @@ check_root SOURCE_DATE_EPOCH,SRCDEST,SRCPKGDEST,PKGDEST,LOGDEST,MAKEFLAGS,PACKAG
 # Pass all arguments after -- right to makepkg
 makechrootpkg_args+=("${@:$OPTIND}")
 
+# Automatically recreate the root chroot if a version mismatch is detected
+CURRENT_CHROOT_VERSION=$(cat "${chroots}/${repo}-${arch}/root/.arch-chroot")
+if [[ -f "${chroots}/${repo}-${arch}/root/.arch-chroot" ]] && [[ "$CURRENT_CHROOT_VERSION" != "$CHROOT_VERSION" ]]; then
+	warning "Recreating chroot '%s' (%s) as it is not at version %s" "${chroots}/${repo}-${arch}/root" "$CURRENT_CHROOT_VERSION" "$CHROOT_VERSION"
+	clean_first=true
+fi
+
 if ${clean_first} || [[ ! -d "${chroots}/${repo}-${arch}" ]]; then
 	msg "Creating chroot for [%s] (%s)..." "${repo}" "${arch}"
 

src/checkpkg.in

@@ -140,7 +140,7 @@ for _pkgname in "${pkgname[@]}"; do
 	bsdtar tf "$TEMPDIR/$oldpkg" | sort > "$TEMPDIR/filelist-$_pkgname-old"
 	bsdtar tf "$pkgfile" | sort > "$TEMPDIR/filelist-$_pkgname"
 
-	diff --side-by-side --suppress-common-lines --width="$COLUMNS" --color=auto "$TEMPDIR/filelist-$_pkgname-old" "$TEMPDIR/filelist-$_pkgname"
+	diff --side-by-side --suppress-common-lines --width="${COLUMNS:-130}" --color=auto "$TEMPDIR/filelist-$_pkgname-old" "$TEMPDIR/filelist-$_pkgname"
 
 	find-libprovides "$TEMPDIR/$oldpkg" 2>/dev/null | sort > "$TEMPDIR/libraries-$_pkgname-old"
 	find-libprovides "$pkgfile" 2>/dev/null | sort > "$TEMPDIR/libraries-$_pkgname"

src/lib/api/archweb.sh

@@ -14,6 +14,8 @@ set -o pipefail
 
 
 archweb_query_all_packages() {
+	local -a pkgbases
+
 	[[ -z ${WORKDIR:-} ]] && setup_workdir
 
 	stat_busy "Query all released packages"
@@ -36,6 +38,7 @@ archweb_query_all_packages() {
 
 archweb_query_maintainer_packages() {
 	local maintainer=$1
+	local -a pkgbases
 
 	[[ -z ${WORKDIR:-} ]] && setup_workdir
 

src/lib/api/gitlab.sh

@@ -119,11 +119,13 @@ gitlab_api_call_paged() {
 
 	local next_page=1
 	local total_pages=1
+	local known_total_pages=1
+	local percentage=100
 
 	while [[ -n "${next_page}" ]]; do
 		percentage=$(( 100 * next_page / total_pages ))
 		printf "📡 Querying GitLab: %s/%s [%s] %%spinner%%" \
-			"${BOLD}${next_page}" "${total_pages}" "${percentage}%${ALL_OFF}"  \
+			"${BOLD}${next_page}" "${known_total_pages}" "${percentage}%${ALL_OFF}"  \
 			> "${tmp_file}"
 		mv "${tmp_file}" "${status_file}"
 
@@ -148,6 +150,15 @@ gitlab_api_call_paged() {
 
 		next_page=$(grep "x-next-page" "${header}" | tr -d '\r' | awk '{ print $2 }')
 		total_pages=$(grep "x-total-pages" "${header}" | tr -d '\r' | awk '{ print $2 }')
+		# The api is not guaranteed to return x-total-pages for larger query results
+		# https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/23931
+		# https://gitlab.com/gitlab-org/gitlab/-/issues/436373
+		if (( total_pages == 0 )); then
+			total_pages=${next_page}
+			known_total_pages="?"
+		else
+			known_total_pages=${total_pages}
+		fi
 	done
 
 	jq --slurp add "${api_workdir}"/result.* > "${outfile}"

src/lib/archroot.sh

@@ -4,7 +4,7 @@
 :
 
 # shellcheck disable=2034
-CHROOT_VERSION='v5'
+CHROOT_VERSION='v6'
 
 ##
 #  usage : check_root $keepenv

src/lib/issue/view.sh

@@ -181,7 +181,7 @@ pkgctl_issue_view() {
 		status_dir=$(mktemp --tmpdir="${WORKDIR}" --directory pkgctl-gitlab-api.XXXXXXXXXX)
 		printf "📡 Querying GitLab issue notes API..." > "${status_dir}/status"
 		term_spinner_start "${status_dir}"
-		if ! output=$(gitlab_project_issue_notes "${project_path}" "${iid}" "${status_dir}/status"); then
+		if ! output=$(gitlab_project_issue_notes "${project_path}" "${iid}" "${status_dir}/status" "sort=asc&order_by=created_at"); then
 			term_spinner_stop "${status_dir}"
 			msg_error "Failed to fetch comments"
 			exit 1

src/lib/repo/clone.sh

@@ -65,6 +65,7 @@ pkgctl_repo_clone() {
 	local CONFIGURE_OPTIONS=()
 	local jobs=
 	jobs=$(nproc)
+	local -a pkgbases
 
 	# variables
 	local command=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}}

src/lib/util/pacman.sh

@@ -33,6 +33,7 @@ update_pacman_repo_cache() {
 get_pacman_repo_from_pkgbuild() {
 	local path=${1:-PKGBUILD}
 	local repo=${2:-multilib}
+	local -a pkgnames
 
 	# shellcheck source=contrib/makepkg/PKGBUILD.proto
 	mapfile -t pkgnames < <(source "${path}"; printf "%s\n" "${pkgname[@]}")
@@ -67,6 +68,7 @@ get_pkgnames_from_repo_pkgbase() {
 	local repo=$1
 	shift
 	local pkgbases=("$@")
+	local -a pkgnames
 
 	# update the pacman repo cache if it doesn't exist yet
 	if [[ ! -d "${_DEVTOOLS_PACMAN_CACHE_DIR}" ]]; then

src/lib/version/check.sh

@@ -292,7 +292,8 @@ get_upstream_version() {
 		opts+=(--keyfile "${keyfile}")
 	fi
 
-	if ! output=$(GIT_TERMINAL_PROMPT=0 nvchecker --file "${config}" --logger json "${opts[@]}" 2>&1 | \
+	if ! output=$(GIT_TERMINAL_PROMPT=0 GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null \
+			nvchecker --file "${config}" --logger json "${opts[@]}" 2>&1 | \
 			jq --raw-output 'select((.level != "debug") and (.event != "ignoring invalid version"))'); then
 		printf "failed to run nvchecker: %s" "${output}"
 		return 1

src/offload-build.in

@@ -114,25 +114,32 @@ ssh "${SSH_OPTS[@]}" -- "$server" "cd ${remote_temp@Q} && bsdtar --strip-compone
 # Run the build command on the server
 msg "Running archbuild"
 # shellcheck disable=SC2145
-ssh "${SSH_OPTS[@]}" -t -- "$server" "cd ${remote_temp@Q} && export LOGDEST="" && ${archbuild_cmd[@]@Q}" || die
-msg "Build complete"
+if ssh "${SSH_OPTS[@]}" -t -- "$server" "cd ${remote_temp@Q} && export LOGDEST="" && ${archbuild_cmd[@]@Q}"; then
+    msg "Build complete"
 
-# Get an array of files that should be downloaded from the server
-mapfile -t files < <(
-    ssh "${SSH_OPTS[@]}" -- "$server" "
-        cd ${remote_temp@Q}"' &&
-        makepkg_user_config="${XDG_CONFIG_HOME:-$HOME/.config}/pacman/makepkg.conf" &&
-        makepkg_config="/usr/share/devtools/makepkg.conf.d/'"${arch}"'.conf" &&
-        if [[ -f /usr/share/devtools/makepkg.conf.d/'"${repo}"'-'"${arch}"'.conf ]]; then
-            makepkg_config="/usr/share/devtools/makepkg.conf.d/'"${repo}"'-'"${arch}"'.conf"
-        fi &&
-        while read -r file; do
-            [[ -f "${file}" ]] && printf "%s\n" "${file}" ||:
-        done < <(makepkg --config <(cat "${makepkg_user_config}" "${makepkg_config}" 2>/dev/null) --packagelist) &&
-        printf "%s\n" '"${remote_temp@Q}/PKGBUILD"'
+    # Get an array of files that should be downloaded from the server
+    mapfile -t files < <(
+        ssh "${SSH_OPTS[@]}" -- "$server" "
+            cd ${remote_temp@Q}"' &&
+            makepkg_user_config="${XDG_CONFIG_HOME:-$HOME/.config}/pacman/makepkg.conf" &&
+            makepkg_config="/usr/share/devtools/makepkg.conf.d/'"${arch}"'.conf" &&
+            if [[ -f /usr/share/devtools/makepkg.conf.d/'"${repo}"'-'"${arch}"'.conf ]]; then
+                makepkg_config="/usr/share/devtools/makepkg.conf.d/'"${repo}"'-'"${arch}"'.conf"
+            fi &&
+            while read -r file; do
+                [[ -f "${file}" ]] && printf "%s\n" "${file}" ||:
+            done < <(makepkg --config <(cat "${makepkg_user_config}" "${makepkg_config}" 2>/dev/null) --packagelist) &&
+            printf "%s\n" '"${remote_temp@Q}/PKGBUILD"'
 
-        find '"${remote_temp@Q}"' -name "*.log"
-')
+            find '"${remote_temp@Q}"' -name "*.log"
+    ')
+else
+    # Build failed, only the logs should be downloaded from the server
+    mapfile -t files < <(
+        ssh "${SSH_OPTS[@]}" -- "$server" '
+            find '"${remote_temp@Q}"' -name "*.log"
+    ')
+fi
 
 
 if (( ${#files[@]} )); then
@@ -142,13 +149,14 @@ if (( ${#files[@]} )); then
     if is_globfile "${TEMPDIR}"/*.log; then
         mv "${TEMPDIR}"/*.log "${LOGDEST:-${PWD}}/"
     fi
-    # missing PKGBUILD download means the build failed
-    if [[ ! -f "${TEMPDIR}/PKGBUILD" ]]; then
+    if is_globfile "${TEMPDIR}"/*.pkg.tar*; then
+        # Building a package may change the PKGBUILD during update_pkgver
+        mv "${TEMPDIR}/PKGBUILD" "${PWD}/"
+        mv "${TEMPDIR}"/*.pkg.tar* "${PKGDEST:-${PWD}}/"
+    else
         error "Build failed, check logs in ${LOGDEST:-${PWD}}"
         exit 1
     fi
-    mv "${TEMPDIR}/PKGBUILD" "${PWD}/"
-    mv "${TEMPDIR}"/*.pkg.tar* "${PKGDEST:-${PWD}}/"
 else
     exit 1
 fi

test/Justfile

@@ -0,0 +1,4 @@
+install:
+  rm -rf src/devtools-local
+  makepkg -f
+  sudo pacman --noconfirm -U $(makepkg --packagelist | head -1)

test/PKGBUILD

@@ -0,0 +1,67 @@
+# Maintainer: Levente Polyak <anthraxx[at]archlinux[dot]org>
+# Contributor: Pierre Schmitz <pierre@archlinux.de>
+
+pkgname=devtools
+branch=master
+pkgver=1.3.1.r4.g79c3162
+pkgrel=1
+pkgdesc='Tools for Arch Linux package maintainers'
+arch=('any')
+license=('GPL')
+url='https://gitlab.archlinux.org/archlinux/devtools'
+depends=(
+  arch-install-scripts
+  awk
+  bash
+  binutils
+  coreutils
+  diffutils
+  fakeroot
+  findutils
+  grep
+  jq
+  openssh
+  parallel
+  rsync
+  sed
+  util-linux
+
+  bzr
+  git
+  mercurial
+  subversion
+)
+makedepends=(
+  asciidoc
+  shellcheck
+)
+optdepends=('btrfs-progs: btrfs support')
+source=(devtools-local::"git+file://$PWD/../.git#branch=${branch}")
+validpgpkeys=(
+  '4AA4767BBC9C4B1D18AE28B77F2D434B9741E8AC' # Pierre Schmitz <pierre@archlinux.org>
+  '86CFFCA918CF3AF47147588051E8B148A9999C34' # Evangelos Foutras <foutrelis@archlinux.org>
+  '8FC15A064950A99DD1BD14DD39E4B877E62EB915' # Sven-Hendrik Haase <svenstaro@archlinux.org>
+  'A2FF3A36AAA56654109064AB19802F8B0D70FC30' # Jan Alexander Steffens (heftig) <heftig@archlinux.org>
+  'B81B051F2D7FC867AAFF35A58DBD63B82072D77A' # Sébastien Luttringer <seblu@archlinux.org>
+  '6645B0A8C7005E78DB1D7864F99FFE0FEAE999BD' # Allan McRae (Developer) <allan@archlinux.org>
+  'E240B57E2C4630BA768E2F26FC1B547C8D8172C8' # Levente Polyak <anthraxx@archlinux.org>
+)
+sha256sums=('SKIP')
+b2sums=('SKIP')
+
+pkgver() {
+  cd ${pkgname}-local
+  git describe --long --tags | sed -E 's,^[^0-9]*,,;s,([^-]*-g),r\1,;s,-,.,g'
+}
+
+build() {
+  cd ${pkgname}-local
+  make BUILDTOOLVER="${epoch}:${pkgver}-${pkgrel}-${arch}" PREFIX=/usr
+}
+
+package() {
+  cd ${pkgname}-local
+  make PREFIX=/usr DESTDIR="${pkgdir}" install
+}
+
+# vim: ts=2 sw=2 et: