PoC: build offline support

Since commit 1d433f6 ("feat(db): confirm list of all packages that will be removed") packages need confirmation by default when being dropped from the db. If we make it to the DB drop phase the package already is pushed to the AUR, so it is safe to remove from the database and not removing it would create a somewhat broken state, so we assume that the package should unconditionally be removed from the db.

Component: pkgctl aur drop-from-repo
Signed-off-by: Christian Heusel <christian@heusel.eu>

.editorconfig

@@ -0,0 +1,15 @@
+# EditorConfig configuration for devtools
+# https://editorconfig.org
+
+# Top-most EditorConfig file
+root = true
+
+# Unix-style newlines without trailing whitespaces, but with a newline
+# ending every file, utf-8 charset, set indent to tabs
+[*]
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+charset = utf-8
+indent_style = tab
+tab_width = 4

Makefile

@@ -1,6 +1,6 @@
 SHELL=/bin/bash -o pipefail
 
-V=1.2.0
+V=1.2.1
 BUILDTOOLVER ?= $(V)
 
 PREFIX = /usr/local

doc/man/pkgctl-build.1.asciidoc

@@ -15,6 +15,8 @@ Description
 Build packages in clean chroot environment, offering various options
 and functionalities to customize the package building process.
 
+By default, chroot environments are located in '/var/lib/archbuild/'.
+
 Build Options
 -------------
 

src/checkpkg.in

@@ -140,7 +140,7 @@ for _pkgname in "${pkgname[@]}"; do
 	bsdtar tf "$TEMPDIR/$oldpkg" | sort > "$TEMPDIR/filelist-$_pkgname-old"
 	bsdtar tf "$pkgfile" | sort > "$TEMPDIR/filelist-$_pkgname"
 
-	sdiff -s "$TEMPDIR/filelist-$_pkgname-old" "$TEMPDIR/filelist-$_pkgname"
+	diff --side-by-side --suppress-common-lines --width="$COLUMNS" --color=auto "$TEMPDIR/filelist-$_pkgname-old" "$TEMPDIR/filelist-$_pkgname"
 
 	find-libprovides "$TEMPDIR/$oldpkg" 2>/dev/null | sort > "$TEMPDIR/libraries-$_pkgname-old"
 	find-libprovides "$pkgfile" 2>/dev/null | sort > "$TEMPDIR/libraries-$_pkgname"

src/commitpkg.in

@@ -7,8 +7,6 @@ _DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@}
 source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh
 # shellcheck source=src/lib/util/srcinfo.sh
 source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/srcinfo.sh
-# shellcheck source=src/lib/state.sh
-source "${_DEVTOOLS_LIBRARY_DIR}"/lib/state.sh
 
 source /usr/share/makepkg/util/util.sh
 
@@ -236,9 +234,6 @@ declare -a uploads
 declare -a commit_arches
 declare -a skip_arches
 
-BUILD_STATE_DIR=$(get_state_folder "build-state")
-state_file=
-
 for _arch in "${arch[@]}"; do
 	if [[ -n $commit_arch && ${_arch} != "$commit_arch" ]]; then
 		skip_arches+=("$_arch")
@@ -252,12 +247,6 @@ for _arch in "${arch[@]}"; do
 			skip_arches+=("$_arch")
 			continue 2
 		fi
-
-		state_file="${BUILD_STATE_DIR}/$(basename "${pkgfile}").txt"
-		if [[ -f "${state_file}" ]] && [[ $(cat "${state_file}") != "${repo}" ]]; then
-			error "%s was not built against '%s', aborting" "${pkgfile}" "${repo}"
-			exit 1
-		fi
 		uploads+=("$pkgfile")
 	done
 

src/lib/aur/drop-from-repo.sh

@@ -165,7 +165,7 @@ pkgctl_aur_drop_from_repo() {
 			warning 'Did not find %s in any repository, please delete manually' "${pkgbase}"
 		else
 			msg2 "  repo: ${pkgrepo}"
-			pkgctl_db_remove "${pkgrepo}" "${pkgbase}"
+			pkgctl_db_remove --noconfirm "${pkgrepo}" "${pkgbase}"
 		fi
 
 		popd >/dev/null

src/lib/auth/login.sh

@@ -63,7 +63,7 @@ pkgctl_auth_login() {
 		esac
 	done
 
-	personal_access_token_url="https://${GITLAB_HOST}/-/profile/personal_access_tokens?name=pkgctl+token&scopes=api,write_repository"
+	personal_access_token_url="https://${GITLAB_HOST}/-/user_settings/personal_access_tokens?name=pkgctl+token&scopes=api,write_repository"
 
     cat <<- _EOF_
 	Logging into ${BOLD}${GITLAB_HOST}${ALL_OFF}

src/lib/build/build.sh

@@ -12,8 +12,6 @@ source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh
 source "${_DEVTOOLS_LIBRARY_DIR}"/lib/db/update.sh
 # shellcheck source=src/lib/release.sh
 source "${_DEVTOOLS_LIBRARY_DIR}"/lib/release.sh
-# shellcheck source=src/lib/state.sh
-source "${_DEVTOOLS_LIBRARY_DIR}"/lib/state.sh
 # shellcheck source=src/lib/util/git.sh
 source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/git.sh
 # shellcheck source=src/lib/util/srcinfo.sh
@@ -57,6 +55,7 @@ pkgctl_build_usage() {
 		    -o, --offload        Build on a remote server and transfer artifacts afterwards
 		    -c, --clean          Recreate the chroot before building
 		    --inspect WHEN       Spawn an interactive shell to inspect the chroot (never, always, failure)
+		    --offline MODE       Run a part of the build process offline (build, check)
 		    -w, --worker SLOT    Name of the worker slot, useful for concurrent builds (disables automatic names)
 		    --nocheck            Do not run the check() function in the PKGBUILD
 
@@ -81,8 +80,8 @@ pkgctl_build_usage() {
 
 		EXAMPLES
 		    $ ${COMMAND}
-		    $ ${COMMAND} --rebuild --staging --message 'libyay 0.42 rebuild' libfoo libbar
+		    $ ${COMMAND} --rebuild --staging --release --message 'libyay 0.42 rebuild' libfoo libbar
-		    $ ${COMMAND} --pkgver 1.42 --release --db-update
+		    $ ${COMMAND} --pkgver=1.42 --release --db-update
 _EOF_
 }
 
@@ -131,7 +130,6 @@ pkgctl_build() {
 	local PKGVER=
 	local PKGREL=
 	local MESSAGE=
-	local BUILD_STATE_DIR=
 
 	local paths=()
 	local BUILD_ARCH=()
@@ -201,6 +199,10 @@ pkgctl_build() {
 				EDIT=1
 				shift
 				;;
+			--offline)
+				MAKECHROOT_OPTIONS+=("-o" "$2")
+				shift 2
+				;;
 			-o|--offload)
 				OFFLOAD=1
 				shift
@@ -307,8 +309,6 @@ pkgctl_build() {
 		fi
 	fi
 
-	BUILD_STATE_DIR=$(get_state_folder "build-state")
-
 	# assign default worker slot
 	if [[ -z ${WORKER_SLOT} ]] && ! WORKER_SLOT="$(tty | sed 's|/dev/pts/||')"; then
 		WORKER_SLOT=$(( RANDOM % $(nproc) + 1 ))
@@ -486,6 +486,8 @@ pkgctl_build() {
 		# shellcheck disable=SC2119
 		write_srcinfo_file
 
+		# test-install (some of) the produced packages
+		if [[ ${INSTALL_TO_HOST} == auto ]] || [[ ${INSTALL_TO_HOST} == all ]]; then
 			# shellcheck disable=2119
 			load_makepkg_config
 
@@ -495,17 +497,14 @@ pkgctl_build() {
 
 			for pkg in "${pkgname[@]}"; do
 				pkg_architecture=$(get_pkg_arch "$pkg")
-			pkgpath=$(realpath "$(printf "%s\n" "${PKGDEST:-.}")")
+				pkgfile=$(realpath "$(printf "%s/%s-%s-%s%s\n" "${PKGDEST:-.}" "$pkg" "$version" "$pkg_architecture" "$PKGEXT")")
-			pkgfile=$(printf "%s-%s-%s%s\n" "$pkg" "$version" "$pkg_architecture" "$PKGEXT")
 
 				# check if we install all packages or if the (split-)package is already installed
 				if [[ ${INSTALL_TO_HOST} == all ]] || ( [[ ${INSTALL_TO_HOST} == auto ]] && pacman -Qq -- "$pkg" &>/dev/null ); then
-				INSTALL_HOST_PACKAGES+=("${pkgpath}/${pkgfile}")
+					INSTALL_HOST_PACKAGES+=("$pkgfile")
 				fi
-
-			# save against which repo we have built the package
-			printf "%s" "${pkgrepo}" > "${BUILD_STATE_DIR}/${pkgfile}.txt"
 			done
+		fi
 
 		# release the build
 		if (( RELEASE )); then

src/lib/common.sh

@@ -120,6 +120,8 @@ print_workdir_error() {
 }
 
 _setup_workdir=false
+# Ensure that there is no outside value for WORKDIR leaking in
+unset WORKDIR
 setup_workdir() {
 	[[ -z ${WORKDIR:-} ]] && WORKDIR=$(mktemp -d --tmpdir "${0##*/}.XXXXXXXXXX")
 	_setup_workdir=true

src/lib/repo/configure.sh

@@ -271,6 +271,7 @@ pkgctl_repo_configure() {
 		if [[ -n $GPGKEY ]]; then
 			git config commit.gpgsign true
 			git config user.signingKey "${GPGKEY}"
+			git config gpg.format openpgp
 		fi
 
 		# set default git exclude

src/lib/state.sh

@@ -1,18 +0,0 @@
-#!/hint/bash
-#
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-[[ -z ${DEVTOOLS_INCLUDE_STATE_SH:-} ]] || return 0
-DEVTOOLS_INCLUDE_STATE_SH=1
-
-set -e
-
-readonly XDG_DEVTOOLS_STATE_DIR="${XDG_STATE_HOME:-$HOME/.local/state}/devtools"
-
-get_state_folder() {
-	local foldername=$1
-	local path="${XDG_DEVTOOLS_STATE_DIR}/${foldername}"
-
-	mkdir --parents -- "$path"
-	printf '%s' "${path}"
-}

src/lib/version/check.sh

@@ -284,7 +284,7 @@ nvchecker_check_error() {
 	local errors
 
 	if ! errors=$(jq --raw-output --exit-status \
-			'select(.level == "error") | "\(.event)" + if .error then ": \(.error)" else "" end' \
+			'select((.level == "error") and (.error != null)) | "\(.event)" + if .error then ": \(.error)" else "" end' \
 			<<< "${result}"); then
 		return 0
 	fi

src/makechrootpkg.in

@@ -40,6 +40,8 @@ bindmounts_ro=()
 bindmounts_rw=()
 bindmounts_tmpfs=()
 
+offline_options=()
+
 copy=$USER
 [[ -n ${SUDO_USER:-} ]] && copy=$SUDO_USER
 [[ -z "$copy" || $copy = root ]] && copy=copy
@@ -80,6 +82,7 @@ usage() {
 	echo '                  Useful for maintaining multiple copies'
 	echo "                  Default: $copy"
 	echo '-n                Run namcap on the package'
+	echo '-o                Run given step offline'
 	echo '-C                Run checkpkg on the package'
 	echo '-T                Build in a temporary directory'
 	echo '-U                Run makepkg as a specified user'
@@ -105,7 +108,7 @@ sync_chroot() {
 		"Locking clean chroot [%s]" "$chrootdir/root"
 
 	stat_busy "Synchronizing chroot copy [%s] -> [%s]" "$chrootdir/root" "$copy"
-	if is_btrfs "$chrootdir" && ! mountpoint -q "$copydir"; then
+	if is_btrfs "$chrootdir" && is_subvolume "$chrootdir/root" && ! mountpoint -q "$copydir"; then
 		subvolume_delete_recursive "$copydir" ||
 			die "Unable to delete subvolume %s" "$copydir"
 		btrfs subvolume snapshot "$chrootdir/root" "$copydir" >/dev/null ||
@@ -203,6 +206,7 @@ EOF
 		declare -p SOURCE_DATE_EPOCH 2>/dev/null || true
 		declare -p BUILDTOOL 2>/dev/null
 		declare -p BUILDTOOLVER 2>/dev/null
+		declare -p offline_options
 		printf '_chrootbuild "$@" || exit\n'
 
 		if (( run_namcap )); then
@@ -222,14 +226,52 @@ _chrootbuild() {
 	# shellcheck source=/dev/null
 	. /etc/profile
 
+	# for in_array
+	. /usr/share/makepkg/util.sh
+
 	# Beware, there are some stupid arbitrary rules on how you can
 	# use "$" in arguments to commands with "sudo -i".  ${foo} or
 	# ${1} is OK, but $foo or $1 isn't.
 	# https://bugzilla.sudo.ws/show_bug.cgi?id=765
+
+	# Run prepare
 	sudo --preserve-env=SOURCE_DATE_EPOCH \
 		--preserve-env=BUILDTOOL \
 		--preserve-env=BUILDTOOLVER \
-		-iu builduser bash -c 'cd /startdir; makepkg "$@"' -bash "$@"
+		-iu builduser bash -c 'cd /startdir; makepkg --nobuild "$@"' -bash "$@"
+
+	if in_array "build" "${offline_options[@]}"; then
+		msg "building offline"
+		# Build offline
+		unshare -n -- sudo --preserve-env=SOURCE_DATE_EPOCH \
+			--preserve-env=BUILDTOOL \
+			--preserve-env=BUILDTOOLVER \
+			-iu builduser \
+			bash -c 'cd /startdir; makepkg --noprepare --noextract --nocheck "$@"' -bash "$@"
+	else
+		sudo --preserve-env=SOURCE_DATE_EPOCH \
+			--preserve-env=BUILDTOOL \
+			--preserve-env=BUILDTOOLVER \
+			-iu builduser \
+			bash -c 'cd /startdir; makepkg --noprepare --noextract --nocheck "$@"' -bash "$@"
+	fi
+
+	if in_array "check" "${offline_options[@]}"; then
+		msg "check offline"
+		# Run tests online
+		unshare -n -- sudo --preserve-env=SOURCE_DATE_EPOCH \
+			--preserve-env=BUILDTOOL \
+			--preserve-env=BUILDTOOLVER \
+			-iu builduser \
+			bash -c 'cd /startdir; makepkg --noprepare --noextract --nobuild "$@"' -bash "$@"
+	else
+		sudo --preserve-env=SOURCE_DATE_EPOCH \
+			--preserve-env=BUILDTOOL \
+			--preserve-env=BUILDTOOLVER \
+			-iu builduser \
+			bash -c 'cd /startdir; makepkg --noprepare --noextract --nobuild "$@"' -bash "$@"
+	fi
+
 	ret=$?
 	case $ret in
 		0|14)
@@ -293,7 +335,7 @@ move_products() {
 }
 # }}}
 
-while getopts 'hcur:I:l:nCTD:d:U:x:t:' arg; do
+while getopts 'hcur:I:l:nCTD:o:d:U:x:t:' arg; do
 	case "$arg" in
 		c) clean_first=1 ;;
 		D) bindmounts_ro+=("--bind-ro=$OPTARG") ;;
@@ -304,6 +346,7 @@ while getopts 'hcur:I:l:nCTD:d:U:x:t:' arg; do
 		I) install_pkgs+=("$OPTARG") ;;
 		l) copy="$OPTARG" ;;
 		n) run_namcap=1; makepkg_args+=(--install) ;;
+		o) offline_options+=("$OPTARG") ;;
 		C) run_checkpkg=1 ;;
 		T) temp_chroot=1; copy+="-$$" ;;
 		U) makepkg_user="$OPTARG" ;;