commitpkg: introduce reuse lint and download

Signed-off-by: Morten Linderud <foxboron@archlinux.org>

Makefile

@@ -1,6 +1,6 @@
 SHELL=/bin/bash -o pipefail
 
-V=1.3.0
+V=1.3.2
 BUILDTOOLVER ?= $(V)
 
 PREFIX = /usr/local
@@ -106,7 +106,7 @@ $(BUILDDIR)/doc/man/%: doc/man/%.asciidoc doc/man/include/footer.asciidoc
 
 conf:
 	@install -d $(BUILDDIR)/makepkg.conf.d
-	@cp -a $(MAKEPKG_CONFIGS) $(BUILDDIR)/makepkg.conf.d
+	@cp -ra $(MAKEPKG_CONFIGS) $(BUILDDIR)/makepkg.conf.d
 	@install -d $(BUILDDIR)/pacman.conf.d
 	@cp -a $(PACMAN_CONFIGS) $(BUILDDIR)/pacman.conf.d
 	@install -d $(BUILDDIR)/git.conf.d
@@ -124,7 +124,7 @@ install: all
 	install -dm0755 $(DESTDIR)$(DATADIR)/lib
 	cp -ra $(BUILDDIR)/lib/* $(DESTDIR)$(DATADIR)/lib
 	cp -a $(BUILDDIR)/git.conf.d -t $(DESTDIR)$(DATADIR)
-	for conf in $(notdir $(MAKEPKG_CONFIGS)); do install -Dm0644 $(BUILDDIR)/makepkg.conf.d/$$conf $(DESTDIR)$(DATADIR)/makepkg.conf.d/$${conf##*/}; done
+	cp -ra $(BUILDDIR)/makepkg.conf.d -t $(DESTDIR)$(DATADIR)
 	for conf in $(notdir $(PACMAN_CONFIGS)); do install -Dm0644 $(BUILDDIR)/pacman.conf.d/$$conf $(DESTDIR)$(DATADIR)/pacman.conf.d/$${conf##*/}; done
 	for a in ${SETARCH_ALIASES}; do install -m0644 $$a -t $(DESTDIR)$(DATADIR)/setarch-aliases.d; done
 	for l in ${COMMITPKG_LINKS}; do ln -sf commitpkg $(DESTDIR)$(PREFIX)/bin/$$l; done
@@ -142,7 +142,7 @@ uninstall:
 	for f in $(notdir $(LIBRARY)); do rm -f $(DESTDIR)$(DATADIR)/lib/$$f; done
 	rm -rf $(DESTDIR)$(DATADIR)/lib
 	rm -rf $(DESTDIR)$(DATADIR)/git.conf.d
-	for conf in $(notdir $(MAKEPKG_CONFIGS)); do rm -f $(DESTDIR)$(DATADIR)/makepkg.conf.d/$${conf##*/}; done
+	rm -rf $(DESTDIR)$(DATADIR)/makepkg.conf.d
 	for conf in $(notdir $(PACMAN_CONFIGS)); do rm -f $(DESTDIR)$(DATADIR)/pacman.conf.d/$${conf##*/}; done
 	for f in $(notdir $(SETARCH_ALIASES)); do rm -f $(DESTDIR)$(DATADIR)/setarch-aliases.d/$$f; done
 	for l in ${COMMITPKG_LINKS}; do rm -f $(DESTDIR)$(PREFIX)/bin/$$l; done
@@ -154,7 +154,6 @@ uninstall:
 	for manfile in $(notdir $(MANS)); do rm -f $(DESTDIR)$(MANDIR)/man$${manfile##*.}/$${manfile}; done;
 	rmdir --ignore-fail-on-non-empty \
 		$(DESTDIR)$(DATADIR)/setarch-aliases.d \
-		$(DESTDIR)$(DATADIR)/makepkg.conf.d \
 		$(DESTDIR)$(DATADIR)/pacman.conf.d \
 		$(DESTDIR)$(DATADIR)
 

config/makepkg/conf.d/fortran.conf

@@ -0,0 +1,18 @@
+#!/hint/bash
+#
+# /etc/makepkg.conf.d/fortran.conf
+#
+
+#########################################################################
+# FORTRAN LANGUAGE SUPPORT
+#########################################################################
+
+# Flags used for the Fortran compiler, similar in spirit to CFLAGS. Read
+# linkman:gfortran[1] for more details on the available flags.
+#FFLAGS="-O2 -pipe"
+#FCFLAGS="$FFLAGS"
+
+# Additional compiler flags appended to `FFLAGS` and `FCFLAGS` for use in debugging. Usually
+# this would include: ``-g''. Read linkman:gfortran[1] for more details on the wide
+# variety of compiler flags available.
+#DEBUG_FFLAGS="-g"

config/makepkg/conf.d/rust.conf

@@ -0,0 +1,19 @@
+#!/hint/bash
+# shellcheck disable=2034
+
+#
+# /etc/makepkg.conf.d/rust.conf
+#
+
+#########################################################################
+# RUST LANGUAGE SUPPORT
+#########################################################################
+
+# Flags used for the Rust compiler, similar in spirit to CFLAGS. Read
+# linkman:rustc[1] for more details on the available flags.
+RUSTFLAGS="-C force-frame-pointers=yes"
+
+# Additional compiler flags appended to `RUSTFLAGS` for use in debugging.
+# Usually this would include: ``-C debuginfo=2''. Read linkman:rustc[1] for
+# more details on the available flags.
+DEBUG_RUSTFLAGS="-C debuginfo=2"

config/makepkg/x86_64.conf.d/fortran.conf

@@ -0,0 +1 @@
+../conf.d/fortran.conf

config/makepkg/x86_64.conf.d/rust.conf

@@ -0,0 +1 @@
+../conf.d/rust.conf

config/makepkg/x86_64_v3.conf.d/fortran.conf

@@ -0,0 +1 @@
+../conf.d/fortran.conf

config/makepkg/x86_64_v3.conf.d/rust.conf

@@ -0,0 +1 @@
+../conf.d/rust.conf

contrib/completion/zsh/_devtools.in

@@ -61,7 +61,7 @@ _pkgctl_build_args=(
 	'--update-checksums[Force computation and update of the checksums (disables auto-detection)]'
 	'(-e --edit)'{-e,--edit}'[Edit the PKGBUILD before building]'
 	'(-r --release)'{-r,--release}'[Automatically commit, tag and release after building]'
-	'(-m --message=)'{-m,--message=}"[Use the given <msg> as the commit message]:message:"
+	'(-m --message)'{-m,--message}"[Use the given <msg> as the commit message]:message:"
 	'(-u --db-update)'{-u,--db-update}'[Automatically update the pacman database as last action]'
 	'(-h --help)'{-h,--help}'[Display usage]'
 	'*:git_dir:_files -/'
@@ -201,8 +201,8 @@ _pkgctl_issue_view_args=(
 )
 
 _pkgctl_release_args=(
-	'(-m --message=)'{-m,--message=}"[Use the given <msg> as the commit message]:message:"
-	'(-r --repo=)'{-r,--repo=}"[Specify a target repository for new packages]:repo:($DEVTOOLS_VALID_REPOS[*])"
+	'(-m --message)'{-m,--message}"[Use the given <msg> as the commit message]:message:"
+	'(-r --repo)'{-r,--repo}"[Specify a target repository for new packages]:repo:($DEVTOOLS_VALID_REPOS[*])"
 	'(-s --staging)'{-s,--staging}'[Release to the staging counterpart of the auto-detected repo]'
 	'(-t --testing)'{-t,--testing}'[Release to the testing counterpart of the auto-detected repo]'
 	'(-u --db-update)'{-u,--db-update}'[Automatically update the pacman database after uploading]'

doc/man/arch-nspawn.1.asciidoc

@@ -23,7 +23,8 @@ Options
 	Location of a pacman config file
 
 *-M* <file>::
-	Location of a makepkg config file
+	Location of a makepkg config file. Specific additions (e.g. build flags for
+	additional languages) can be placed in '<file>.d/*.conf'.
 
 *-c* <dir>::
 	Set pacman cache, if no directory is specified the passed pacman.conf's cachedir is used with a fallback to '/etc/pacman.conf'

doc/man/makerepropkg.1.asciidoc

@@ -49,7 +49,8 @@ Options
 	Set the pacman cache directory.
 
 *-M* <file>::
-	Location of a makepkg config file.
+	Location of a makepkg config file. Specific additions (e.g. build flags for
+	additional languages) can be placed in '<file>.d/*.conf'.
 
 *-l* <chroot>::
 	The directory name to use as the chroot namespace

src/arch-nspawn.in

@@ -112,7 +112,13 @@ copy_hostconf () {
 	[[ -n $host_cachemirrors ]] && printf 'CacheServer = %s\n' "${host_cachemirrors[@]}" >>"$working_dir/etc/pacman.d/mirrorlist"
 
 	[[ -n $pac_conf ]] && cp "$pac_conf" "$working_dir/etc/pacman.conf"
-	[[ -n $makepkg_conf ]] && cp "$makepkg_conf" "$working_dir/etc/makepkg.conf"
+	if [[ -n $makepkg_conf ]]; then
+		cp "$makepkg_conf" "$working_dir/etc/makepkg.conf"
+		if [[ -d "${makepkg_conf}.d" ]] && is_globfile "${makepkg_conf}.d"/*.conf; then
+			mkdir --parents "$working_dir/etc/makepkg.conf.d/"
+			cp "${makepkg_conf}.d/"*.conf "$working_dir/etc/makepkg.conf.d/"
+		fi
+	fi
 
 	local file
 	for file in "${files[@]}"; do

src/archbuild.in

@@ -79,6 +79,13 @@ check_root SOURCE_DATE_EPOCH,SRCDEST,SRCPKGDEST,PKGDEST,LOGDEST,MAKEFLAGS,PACKAG
 # Pass all arguments after -- right to makepkg
 makechrootpkg_args+=("${@:$OPTIND}")
 
+# Automatically recreate the root chroot if a version mismatch is detected
+CURRENT_CHROOT_VERSION=$(cat "${chroots}/${repo}-${arch}/root/.arch-chroot")
+if [[ -f "${chroots}/${repo}-${arch}/root/.arch-chroot" ]] && [[ "$CURRENT_CHROOT_VERSION" != "$CHROOT_VERSION" ]]; then
+	warning "Recreating chroot '%s' (%s) as it is not at version %s" "${chroots}/${repo}-${arch}/root" "$CURRENT_CHROOT_VERSION" "$CHROOT_VERSION"
+	clean_first=true
+fi
+
 if ${clean_first} || [[ ! -d "${chroots}/${repo}-${arch}" ]]; then
 	msg "Creating chroot for [%s] (%s)..." "${repo}" "${arch}"
 

src/checkpkg.in

@@ -140,7 +140,7 @@ for _pkgname in "${pkgname[@]}"; do
 	bsdtar tf "$TEMPDIR/$oldpkg" | sort > "$TEMPDIR/filelist-$_pkgname-old"
 	bsdtar tf "$pkgfile" | sort > "$TEMPDIR/filelist-$_pkgname"
 
-	diff --side-by-side --suppress-common-lines --width="$COLUMNS" --color=auto "$TEMPDIR/filelist-$_pkgname-old" "$TEMPDIR/filelist-$_pkgname"
+	diff --side-by-side --suppress-common-lines --width="${COLUMNS:-130}" --color=auto "$TEMPDIR/filelist-$_pkgname-old" "$TEMPDIR/filelist-$_pkgname"
 
 	find-libprovides "$TEMPDIR/$oldpkg" 2>/dev/null | sort > "$TEMPDIR/libraries-$_pkgname-old"
 	find-libprovides "$pkgfile" 2>/dev/null | sort > "$TEMPDIR/libraries-$_pkgname"

src/commitpkg.in

@@ -120,6 +120,14 @@ if (( ${#validpgpkeys[@]} != 0 )); then
 	git add --force -- keys/pgp/*
 fi
 
+# Allow us to disable REUSE
+_RUN_REUSE=${_RUN_REUSE:-1}
+if ((_RUN_REUSE)); then
+	reuse download
+	reuse lint
+	git add --force -- LICENSES/*
+fi
+
 # find files which should be under source control
 needsversioning=(PKGBUILD)
 for s in "${source[@]}"; do

src/lib/api/archweb.sh

@@ -14,6 +14,8 @@ set -o pipefail
 
 
 archweb_query_all_packages() {
+	local -a pkgbases
+
 	[[ -z ${WORKDIR:-} ]] && setup_workdir
 
 	stat_busy "Query all released packages"
@@ -36,6 +38,7 @@ archweb_query_all_packages() {
 
 archweb_query_maintainer_packages() {
 	local maintainer=$1
+	local -a pkgbases
 
 	[[ -z ${WORKDIR:-} ]] && setup_workdir
 

src/lib/api/gitlab.sh

@@ -119,11 +119,13 @@ gitlab_api_call_paged() {
 
 	local next_page=1
 	local total_pages=1
+	local known_total_pages=1
+	local percentage=100
 
 	while [[ -n "${next_page}" ]]; do
 		percentage=$(( 100 * next_page / total_pages ))
 		printf "📡 Querying GitLab: %s/%s [%s] %%spinner%%" \
-			"${BOLD}${next_page}" "${total_pages}" "${percentage}%${ALL_OFF}"  \
+			"${BOLD}${next_page}" "${known_total_pages}" "${percentage}%${ALL_OFF}"  \
 			> "${tmp_file}"
 		mv "${tmp_file}" "${status_file}"
 
@@ -148,6 +150,15 @@ gitlab_api_call_paged() {
 
 		next_page=$(grep "x-next-page" "${header}" | tr -d '\r' | awk '{ print $2 }')
 		total_pages=$(grep "x-total-pages" "${header}" | tr -d '\r' | awk '{ print $2 }')
+		# The api is not guaranteed to return x-total-pages for larger query results
+		# https://gitlab.com/gitlab-org/gitlab-foss/-/merge_requests/23931
+		# https://gitlab.com/gitlab-org/gitlab/-/issues/436373
+		if (( total_pages == 0 )); then
+			total_pages=${next_page}
+			known_total_pages="?"
+		else
+			known_total_pages=${total_pages}
+		fi
 	done
 
 	jq --slurp add "${api_workdir}"/result.* > "${outfile}"

src/lib/archroot.sh

@@ -4,7 +4,7 @@
 :
 
 # shellcheck disable=2034
-CHROOT_VERSION='v5'
+CHROOT_VERSION='v6'
 
 ##
 #  usage : check_root $keepenv

src/lib/build/build.sh

@@ -42,10 +42,10 @@ pkgctl_build_usage() {
 
 		Build packages inside a clean chroot
 
-		When a new pkgver is set using the appropriate PKGBUILD options the
-		checksums are automatically updated.
+		Build packages in clean chroot environment, offering various options
+		and functionalities to customize the package building process.
 
-		TODO
+		By default, chroot environments are located in /var/lib/archbuild/.
 
 		BUILD OPTIONS
 		    --arch ARCH          Specify architectures to build for (disables auto-detection)

src/lib/common.sh

@@ -18,6 +18,9 @@ export LANG=C.UTF-8
 # Avoid systemd trying to color the terminal on systemd-nspawn
 export SYSTEMD_TINT_BACKGROUND=no
 
+# Avoid diffoscope looking at remote debug info through readelf
+unset DEBUGINFOD_URLS
+
 # Set buildtool properties
 export BUILDTOOL=devtools
 export BUILDTOOLVER=@buildtoolver@

src/lib/issue/view.sh

@@ -181,7 +181,7 @@ pkgctl_issue_view() {
 		status_dir=$(mktemp --tmpdir="${WORKDIR}" --directory pkgctl-gitlab-api.XXXXXXXXXX)
 		printf "📡 Querying GitLab issue notes API..." > "${status_dir}/status"
 		term_spinner_start "${status_dir}"
-		if ! output=$(gitlab_project_issue_notes "${project_path}" "${iid}" "${status_dir}/status"); then
+		if ! output=$(gitlab_project_issue_notes "${project_path}" "${iid}" "${status_dir}/status" "sort=asc&order_by=created_at"); then
 			term_spinner_stop "${status_dir}"
 			msg_error "Failed to fetch comments"
 			exit 1

src/lib/repo/clone.sh

@@ -65,6 +65,7 @@ pkgctl_repo_clone() {
 	local CONFIGURE_OPTIONS=()
 	local jobs=
 	jobs=$(nproc)
+	local -a pkgbases
 
 	# variables
 	local command=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}}

src/lib/util/pacman.sh

@@ -33,6 +33,7 @@ update_pacman_repo_cache() {
 get_pacman_repo_from_pkgbuild() {
 	local path=${1:-PKGBUILD}
 	local repo=${2:-multilib}
+	local -a pkgnames
 
 	# shellcheck source=contrib/makepkg/PKGBUILD.proto
 	mapfile -t pkgnames < <(source "${path}"; printf "%s\n" "${pkgname[@]}")
@@ -67,6 +68,7 @@ get_pkgnames_from_repo_pkgbase() {
 	local repo=$1
 	shift
 	local pkgbases=("$@")
+	local -a pkgnames
 
 	# update the pacman repo cache if it doesn't exist yet
 	if [[ ! -d "${_DEVTOOLS_PACMAN_CACHE_DIR}" ]]; then

src/lib/version/check.sh

@@ -292,7 +292,8 @@ get_upstream_version() {
 		opts+=(--keyfile "${keyfile}")
 	fi
 
-	if ! output=$(GIT_TERMINAL_PROMPT=0 nvchecker --file "${config}" --logger json "${opts[@]}" 2>&1 | \
+	if ! output=$(GIT_TERMINAL_PROMPT=0 GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null \
+			nvchecker --file "${config}" --logger json "${opts[@]}" 2>&1 | \
 			jq --raw-output 'select((.level != "debug") and (.event != "ignoring invalid version"))'); then
 		printf "failed to run nvchecker: %s" "${output}"
 		return 1

src/makerepropkg.in

@@ -93,14 +93,36 @@ get_makepkg_conf() {
     local fname=${1}
     local arch="${2}"
     local makepkg_conf="${3}"
+
     if ! buildtool_file=$(get_pkgfile "${fname}"); then
        error "failed to retrieve ${fname}"
        return 1
     fi
-    msg2 "using makepkg.conf from ${fname}"
-	if ! bsdtar xOqf "${buildtool_file/file:\/\//}" "usr/share/devtools/makepkg.conf.d/${arch}.conf" > "${makepkg_conf}"; then
-		bsdtar xOqf "${buildtool_file/file:\/\//}" "usr/share/devtools/makepkg-${arch}.conf" > "${makepkg_conf}"
+    buildtool_file="${buildtool_file/file:\/\//}"
+    msg "using makepkg.conf from ${fname}"
+
+    # try to handle config of legacy devtools
+    if bsdtar --list --file "${buildtool_file}" "usr/share/devtools/makepkg-${arch}.conf" &>/dev/null; then
+        bsdtar --extract --to-stdout --fast-read --file "${buildtool_file}" "usr/share/devtools/makepkg-${arch}.conf" > "${makepkg_conf}"
+        return $?
     fi
+
+    msg2 "extracting ${arch}.conf from devtools archive"
+    if ! bsdtar --extract --to-stdout --fast-read --file "${buildtool_file}" "usr/share/devtools/makepkg.conf.d/${arch}.conf" > "${makepkg_conf}"; then
+        error "failed to extract 'usr/share/devtools/makepkg.conf.d/${arch}.conf' from devtools archive"
+        return 1
+    fi
+
+    mkdir --parents "${makepkg_conf}.d"
+    if bsdtar --list --file "${buildtool_file}" "usr/share/devtools/makepkg.conf.d/conf.d" &>/dev/null; then
+        msg2 "extracting conf.d from devtools archive"
+        bsdtar --extract --file "${buildtool_file}" --cd "${makepkg_conf}.d" --strip-components 4 "usr/share/devtools/makepkg.conf.d/conf.d"
+    fi
+    if bsdtar --list --file "${buildtool_file}" "usr/share/devtools/makepkg.conf.d/${arch}.conf.d" &>/dev/null; then
+        msg2 "extracting ${arch}.conf.d from devtools archive"
+        bsdtar --extract --file "${buildtool_file}" --cd "${makepkg_conf}.d" --strip-components 4 "usr/share/devtools/makepkg.conf.d/${arch}.conf.d"
+    fi
+
     return 0
 }
 

src/offload-build.in

@@ -114,7 +114,7 @@ ssh "${SSH_OPTS[@]}" -- "$server" "cd ${remote_temp@Q} && bsdtar --strip-compone
 # Run the build command on the server
 msg "Running archbuild"
 # shellcheck disable=SC2145
-ssh "${SSH_OPTS[@]}" -t -- "$server" "cd ${remote_temp@Q} && export LOGDEST="" && ${archbuild_cmd[@]@Q}" || die
+if ssh "${SSH_OPTS[@]}" -t -- "$server" "cd ${remote_temp@Q} && export LOGDEST="" && ${archbuild_cmd[@]@Q}"; then
     msg "Build complete"
 
     # Get an array of files that should be downloaded from the server
@@ -133,6 +133,13 @@ mapfile -t files < <(
 
             find '"${remote_temp@Q}"' -name "*.log"
     ')
+else
+    # Build failed, only the logs should be downloaded from the server
+    mapfile -t files < <(
+        ssh "${SSH_OPTS[@]}" -- "$server" '
+            find '"${remote_temp@Q}"' -name "*.log"
+    ')
+fi
 
 
 if (( ${#files[@]} )); then
@@ -142,13 +149,14 @@ if (( ${#files[@]} )); then
     if is_globfile "${TEMPDIR}"/*.log; then
         mv "${TEMPDIR}"/*.log "${LOGDEST:-${PWD}}/"
     fi
-    # missing PKGBUILD download means the build failed
-    if [[ ! -f "${TEMPDIR}/PKGBUILD" ]]; then
-        error "Build failed, check logs in ${LOGDEST:-${PWD}}"
-        exit 1
-    fi
+    if is_globfile "${TEMPDIR}"/*.pkg.tar*; then
+        # Building a package may change the PKGBUILD during update_pkgver
         mv "${TEMPDIR}/PKGBUILD" "${PWD}/"
         mv "${TEMPDIR}"/*.pkg.tar* "${PKGDEST:-${PWD}}/"
+    else
+        error "Build failed, check logs in ${LOGDEST:-${PWD}}"
+        exit 1
+    fi
 else
     exit 1
 fi