feat(version): support combiner source for version check commands

This allows more flexibility and chaining by being able to define
multiple sources per pkgbase and chain them together to achieve the
final result. This may also be helpful to combine multiple from/to
patterns into a chain to achieve the final pkgver to compare against.

To utilize the combiner source, the `pkgbase` section must be declared
as the combiner source. Additionally, individual sections should be
added using a quoted table key consisting of the `pkgbase` followed by
the stage name, separated by double colons.
For example: `["sudo:stage1"]`.

Fixes #216

Component: pkgctl version check
Signed-off-by: Levente Polyak <anthraxx@archlinux.org>

Makefile

@@ -1,6 +1,6 @@
 SHELL=/bin/bash -o pipefail
 
-V=1.3.0
+V=1.3.1
 BUILDTOOLVER ?= $(V)
 
 PREFIX = /usr/local

doc/man/pkgctl-version.1.asciidoc

@@ -39,6 +39,14 @@ placed in the `$XDG_CONFIG_HOME`/nvchecker` directory. This keyfile is
 used for providing the necessary authentication tokens required for
 accessing the GitHub or GitLab API.
 
+Combiner Source
+---------------
+
+To utilize the combiner source, the `pkgbase` section must be declared as the
+combiner source. Additionally, individual sections should be added using a
+quoted table key consisting of the `pkgbase` followed by the stage name,
+separated by double colons. For example: `["sudo:stage1"]`.
+
 Options
 -------
 

src/checkpkg.in

@@ -140,7 +140,7 @@ for _pkgname in "${pkgname[@]}"; do
 	bsdtar tf "$TEMPDIR/$oldpkg" | sort > "$TEMPDIR/filelist-$_pkgname-old"
 	bsdtar tf "$pkgfile" | sort > "$TEMPDIR/filelist-$_pkgname"
 
-	diff --side-by-side --suppress-common-lines --width="$COLUMNS" --color=auto "$TEMPDIR/filelist-$_pkgname-old" "$TEMPDIR/filelist-$_pkgname"
+	diff --side-by-side --suppress-common-lines --width="${COLUMNS:-130}" --color=auto "$TEMPDIR/filelist-$_pkgname-old" "$TEMPDIR/filelist-$_pkgname"
 
 	find-libprovides "$TEMPDIR/$oldpkg" 2>/dev/null | sort > "$TEMPDIR/libraries-$_pkgname-old"
 	find-libprovides "$pkgfile" 2>/dev/null | sort > "$TEMPDIR/libraries-$_pkgname"

src/lib/api/archweb.sh

@@ -14,6 +14,8 @@ set -o pipefail
 
 
 archweb_query_all_packages() {
+	local -a pkgbases
+
 	[[ -z ${WORKDIR:-} ]] && setup_workdir
 
 	stat_busy "Query all released packages"
@@ -36,6 +38,7 @@ archweb_query_all_packages() {
 
 archweb_query_maintainer_packages() {
 	local maintainer=$1
+	local -a pkgbases
 
 	[[ -z ${WORKDIR:-} ]] && setup_workdir
 

src/lib/api/gitlab.sh

@@ -119,11 +119,13 @@ gitlab_api_call_paged() {
 
 	local next_page=1
 	local total_pages=1
+	local known_total_pages=1
+	local percentage=100
 
 	while [[ -n "${next_page}" ]]; do
 		percentage=$(( 100 * next_page / total_pages ))
 		printf "📡 Querying GitLab: %s/%s [%s] %%spinner%%" \
-			"${BOLD}${next_page}" "${total_pages}" "${percentage}%${ALL_OFF}"  \
+			"${BOLD}${next_page}" "${known_total_pages}" "${percentage}%${ALL_OFF}"  \
 			> "${tmp_file}"
 		mv "${tmp_file}" "${status_file}"
 
@@ -148,6 +150,15 @@ gitlab_api_call_paged() {
 
 		next_page=$(grep "x-next-page" "${header}" | tr -d '\r' | awk '{ print $2 }')
 		total_pages=$(grep "x-total-pages" "${header}" | tr -d '\r' | awk '{ print $2 }')
+		# The api is not guaranteed to return x-total-pages for larger query results
+		# https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/23931
+		# https://gitlab.com/gitlab-org/gitlab/-/issues/436373
+		if (( total_pages == 0 )); then
+			total_pages=${next_page}
+			known_total_pages="?"
+		else
+			known_total_pages=${total_pages}
+		fi
 	done
 
 	jq --slurp add "${api_workdir}"/result.* > "${outfile}"

src/lib/issue/view.sh

@@ -181,7 +181,7 @@ pkgctl_issue_view() {
 		status_dir=$(mktemp --tmpdir="${WORKDIR}" --directory pkgctl-gitlab-api.XXXXXXXXXX)
 		printf "📡 Querying GitLab issue notes API..." > "${status_dir}/status"
 		term_spinner_start "${status_dir}"
-		if ! output=$(gitlab_project_issue_notes "${project_path}" "${iid}" "${status_dir}/status"); then
+		if ! output=$(gitlab_project_issue_notes "${project_path}" "${iid}" "${status_dir}/status" "sort=asc&order_by=created_at"); then
 			term_spinner_stop "${status_dir}"
 			msg_error "Failed to fetch comments"
 			exit 1

src/lib/repo/clone.sh

@@ -65,6 +65,7 @@ pkgctl_repo_clone() {
 	local CONFIGURE_OPTIONS=()
 	local jobs=
 	jobs=$(nproc)
+	local -a pkgbases
 
 	# variables
 	local command=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}}

src/lib/util/pacman.sh

@@ -33,6 +33,7 @@ update_pacman_repo_cache() {
 get_pacman_repo_from_pkgbuild() {
 	local path=${1:-PKGBUILD}
 	local repo=${2:-multilib}
+	local -a pkgnames
 
 	# shellcheck source=contrib/makepkg/PKGBUILD.proto
 	mapfile -t pkgnames < <(source "${path}"; printf "%s\n" "${pkgname[@]}")
@@ -67,6 +68,7 @@ get_pkgnames_from_repo_pkgbase() {
 	local repo=$1
 	shift
 	local pkgbases=("$@")
+	local -a pkgnames
 
 	# update the pacman repo cache if it doesn't exist yet
 	if [[ ! -d "${_DEVTOOLS_PACMAN_CACHE_DIR}" ]]; then

src/lib/version/check.sh

@@ -292,7 +292,8 @@ get_upstream_version() {
 		opts+=(--keyfile "${keyfile}")
 	fi
 
-	if ! output=$(GIT_TERMINAL_PROMPT=0 nvchecker --file "${config}" --logger json "${opts[@]}" 2>&1 | \
+	if ! output=$(GIT_TERMINAL_PROMPT=0 GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null \
+			nvchecker --file "${config}" --logger json "${opts[@]}" 2>&1 | \
 			jq --raw-output 'select((.level != "debug") and (.event != "ignoring invalid version"))'); then
 		printf "failed to run nvchecker: %s" "${output}"
 		return 1
@@ -303,6 +304,11 @@ get_upstream_version() {
 		return 1
 	fi
 
+	if ! output=$(jq --raw-output --exit-status 'select(.name == "'"${pkgbase}"'")' <<< "${output}"); then
+		printf "failed to select pkgbase result from output"
+		return 1
+	fi
+
 	if ! upstream_version=$(jq --raw-output --exit-status '.version' <<< "${output}"); then
 		printf "failed to select version from result"
 		return 1
@@ -345,7 +351,7 @@ nvchecker_check_config() {
 	fi
 
 	# check if the config contains any section other than pkgbase
-	if [[ -n ${pkgbase} ]] && property=$(grep --max-count=1 --perl-regexp "^\\[(?!\"?${pkgbase//+/\\+}\"?\\]).+\\]" < "${config}"); then
+	if [[ -n ${pkgbase} ]] && property=$(grep --max-count=1 --perl-regexp "^\\[(?!\"?${pkgbase//+/\\+}(:.+)?\"?\\]).+\\]" < "${config}"); then
 		printf "non-pkgbase section not supported in %s: %s" "${config}" "${property}"
 		return 1
 	fi

src/offload-build.in

@@ -114,25 +114,32 @@ ssh "${SSH_OPTS[@]}" -- "$server" "cd ${remote_temp@Q} && bsdtar --strip-compone
 # Run the build command on the server
 msg "Running archbuild"
 # shellcheck disable=SC2145
-ssh "${SSH_OPTS[@]}" -t -- "$server" "cd ${remote_temp@Q} && export LOGDEST="" && ${archbuild_cmd[@]@Q}" || die
-msg "Build complete"
+if ssh "${SSH_OPTS[@]}" -t -- "$server" "cd ${remote_temp@Q} && export LOGDEST="" && ${archbuild_cmd[@]@Q}"; then
+    msg "Build complete"
 
-# Get an array of files that should be downloaded from the server
-mapfile -t files < <(
-    ssh "${SSH_OPTS[@]}" -- "$server" "
-        cd ${remote_temp@Q}"' &&
-        makepkg_user_config="${XDG_CONFIG_HOME:-$HOME/.config}/pacman/makepkg.conf" &&
-        makepkg_config="/usr/share/devtools/makepkg.conf.d/'"${arch}"'.conf" &&
-        if [[ -f /usr/share/devtools/makepkg.conf.d/'"${repo}"'-'"${arch}"'.conf ]]; then
-            makepkg_config="/usr/share/devtools/makepkg.conf.d/'"${repo}"'-'"${arch}"'.conf"
-        fi &&
-        while read -r file; do
-            [[ -f "${file}" ]] && printf "%s\n" "${file}" ||:
-        done < <(makepkg --config <(cat "${makepkg_user_config}" "${makepkg_config}" 2>/dev/null) --packagelist) &&
-        printf "%s\n" '"${remote_temp@Q}/PKGBUILD"'
+    # Get an array of files that should be downloaded from the server
+    mapfile -t files < <(
+        ssh "${SSH_OPTS[@]}" -- "$server" "
+            cd ${remote_temp@Q}"' &&
+            makepkg_user_config="${XDG_CONFIG_HOME:-$HOME/.config}/pacman/makepkg.conf" &&
+            makepkg_config="/usr/share/devtools/makepkg.conf.d/'"${arch}"'.conf" &&
+            if [[ -f /usr/share/devtools/makepkg.conf.d/'"${repo}"'-'"${arch}"'.conf ]]; then
+                makepkg_config="/usr/share/devtools/makepkg.conf.d/'"${repo}"'-'"${arch}"'.conf"
+            fi &&
+            while read -r file; do
+                [[ -f "${file}" ]] && printf "%s\n" "${file}" ||:
+            done < <(makepkg --config <(cat "${makepkg_user_config}" "${makepkg_config}" 2>/dev/null) --packagelist) &&
+            printf "%s\n" '"${remote_temp@Q}/PKGBUILD"'
 
-        find '"${remote_temp@Q}"' -name "*.log"
-')
+            find '"${remote_temp@Q}"' -name "*.log"
+    ')
+else
+    # Build failed, only the logs should be downloaded from the server
+    mapfile -t files < <(
+        ssh "${SSH_OPTS[@]}" -- "$server" '
+            find '"${remote_temp@Q}"' -name "*.log"
+    ')
+fi
 
 
 if (( ${#files[@]} )); then
@@ -142,13 +149,14 @@ if (( ${#files[@]} )); then
     if is_globfile "${TEMPDIR}"/*.log; then
         mv "${TEMPDIR}"/*.log "${LOGDEST:-${PWD}}/"
     fi
-    # missing PKGBUILD download means the build failed
-    if [[ ! -f "${TEMPDIR}/PKGBUILD" ]]; then
+    if is_globfile "${TEMPDIR}"/*.pkg.tar*; then
+        # Building a package may change the PKGBUILD during update_pkgver
+        mv "${TEMPDIR}/PKGBUILD" "${PWD}/"
+        mv "${TEMPDIR}"/*.pkg.tar* "${PKGDEST:-${PWD}}/"
+    else
         error "Build failed, check logs in ${LOGDEST:-${PWD}}"
         exit 1
     fi
-    mv "${TEMPDIR}/PKGBUILD" "${PWD}/"
-    mv "${TEMPDIR}"/*.pkg.tar* "${PKGDEST:-${PWD}}/"
 else
     exit 1
 fi