PoC: build offline support

Since commit 1d433f6 ("feat(db): confirm list of all packages that will be removed") packages need confirmation by default when being dropped from the db. If we make it to the DB drop phase the package already is pushed to the AUR, so it is safe to remove from the database and not removing it would create a somewhat broken state, so we assume that the package should unconditionally be removed from the db.

Component: pkgctl aur drop-from-repo
Signed-off-by: Christian Heusel <christian@heusel.eu>

doc/man/pkgctl-build.1.asciidoc

@@ -15,6 +15,8 @@ Description
 Build packages in clean chroot environment, offering various options
 and functionalities to customize the package building process.
 
+By default, chroot environments are located in '/var/lib/archbuild/'.
+
 Build Options
 -------------
 

src/checkpkg.in

@@ -140,7 +140,7 @@ for _pkgname in "${pkgname[@]}"; do
 	bsdtar tf "$TEMPDIR/$oldpkg" | sort > "$TEMPDIR/filelist-$_pkgname-old"
 	bsdtar tf "$pkgfile" | sort > "$TEMPDIR/filelist-$_pkgname"
 
-	sdiff -s "$TEMPDIR/filelist-$_pkgname-old" "$TEMPDIR/filelist-$_pkgname"
+	diff --side-by-side --suppress-common-lines --width="$COLUMNS" --color=auto "$TEMPDIR/filelist-$_pkgname-old" "$TEMPDIR/filelist-$_pkgname"
 
 	find-libprovides "$TEMPDIR/$oldpkg" 2>/dev/null | sort > "$TEMPDIR/libraries-$_pkgname-old"
 	find-libprovides "$pkgfile" 2>/dev/null | sort > "$TEMPDIR/libraries-$_pkgname"

src/lib/aur/drop-from-repo.sh

@@ -165,7 +165,7 @@ pkgctl_aur_drop_from_repo() {
 			warning 'Did not find %s in any repository, please delete manually' "${pkgbase}"
 		else
 			msg2 "  repo: ${pkgrepo}"
-			pkgctl_db_remove "${pkgrepo}" "${pkgbase}"
+			pkgctl_db_remove --noconfirm "${pkgrepo}" "${pkgbase}"
 		fi
 
 		popd >/dev/null

src/lib/auth/login.sh

@@ -63,7 +63,7 @@ pkgctl_auth_login() {
 		esac
 	done
 
-	personal_access_token_url="https://${GITLAB_HOST}/-/profile/personal_access_tokens?name=pkgctl+token&scopes=api,write_repository"
+	personal_access_token_url="https://${GITLAB_HOST}/-/user_settings/personal_access_tokens?name=pkgctl+token&scopes=api,write_repository"
 
     cat <<- _EOF_
 	Logging into ${BOLD}${GITLAB_HOST}${ALL_OFF}

src/lib/build/build.sh

@@ -55,6 +55,7 @@ pkgctl_build_usage() {
 		    -o, --offload        Build on a remote server and transfer artifacts afterwards
 		    -c, --clean          Recreate the chroot before building
 		    --inspect WHEN       Spawn an interactive shell to inspect the chroot (never, always, failure)
+		    --offline MODE       Run a part of the build process offline (build, check)
 		    -w, --worker SLOT    Name of the worker slot, useful for concurrent builds (disables automatic names)
 		    --nocheck            Do not run the check() function in the PKGBUILD
 
@@ -79,8 +80,8 @@ pkgctl_build_usage() {
 
 		EXAMPLES
 		    $ ${COMMAND}
-		    $ ${COMMAND} --rebuild --staging --message 'libyay 0.42 rebuild' libfoo libbar
+		    $ ${COMMAND} --rebuild --staging --release --message 'libyay 0.42 rebuild' libfoo libbar
-		    $ ${COMMAND} --pkgver 1.42 --release --db-update
+		    $ ${COMMAND} --pkgver=1.42 --release --db-update
 _EOF_
 }
 
@@ -198,6 +199,10 @@ pkgctl_build() {
 				EDIT=1
 				shift
 				;;
+			--offline)
+				MAKECHROOT_OPTIONS+=("-o" "$2")
+				shift 2
+				;;
 			-o|--offload)
 				OFFLOAD=1
 				shift

src/lib/repo/configure.sh

@@ -271,6 +271,7 @@ pkgctl_repo_configure() {
 		if [[ -n $GPGKEY ]]; then
 			git config commit.gpgsign true
 			git config user.signingKey "${GPGKEY}"
+			git config gpg.format openpgp
 		fi
 
 		# set default git exclude

src/lib/version/check.sh

@@ -284,7 +284,7 @@ nvchecker_check_error() {
 	local errors
 
 	if ! errors=$(jq --raw-output --exit-status \
-			'select(.level == "error") | "\(.event)" + if .error then ": \(.error)" else "" end' \
+			'select((.level == "error") and (.error != null)) | "\(.event)" + if .error then ": \(.error)" else "" end' \
 			<<< "${result}"); then
 		return 0
 	fi

src/makechrootpkg.in

@@ -40,6 +40,8 @@ bindmounts_ro=()
 bindmounts_rw=()
 bindmounts_tmpfs=()
 
+offline_options=()
+
 copy=$USER
 [[ -n ${SUDO_USER:-} ]] && copy=$SUDO_USER
 [[ -z "$copy" || $copy = root ]] && copy=copy
@@ -80,6 +82,7 @@ usage() {
 	echo '                  Useful for maintaining multiple copies'
 	echo "                  Default: $copy"
 	echo '-n                Run namcap on the package'
+	echo '-o                Run given step offline'
 	echo '-C                Run checkpkg on the package'
 	echo '-T                Build in a temporary directory'
 	echo '-U                Run makepkg as a specified user'
@@ -105,7 +108,7 @@ sync_chroot() {
 		"Locking clean chroot [%s]" "$chrootdir/root"
 
 	stat_busy "Synchronizing chroot copy [%s] -> [%s]" "$chrootdir/root" "$copy"
-	if is_btrfs "$chrootdir" && ! mountpoint -q "$copydir"; then
+	if is_btrfs "$chrootdir" && is_subvolume "$chrootdir/root" && ! mountpoint -q "$copydir"; then
 		subvolume_delete_recursive "$copydir" ||
 			die "Unable to delete subvolume %s" "$copydir"
 		btrfs subvolume snapshot "$chrootdir/root" "$copydir" >/dev/null ||
@@ -203,6 +206,7 @@ EOF
 		declare -p SOURCE_DATE_EPOCH 2>/dev/null || true
 		declare -p BUILDTOOL 2>/dev/null
 		declare -p BUILDTOOLVER 2>/dev/null
+		declare -p offline_options
 		printf '_chrootbuild "$@" || exit\n'
 
 		if (( run_namcap )); then
@@ -222,14 +226,52 @@ _chrootbuild() {
 	# shellcheck source=/dev/null
 	. /etc/profile
 
+	# for in_array
+	. /usr/share/makepkg/util.sh
+
 	# Beware, there are some stupid arbitrary rules on how you can
 	# use "$" in arguments to commands with "sudo -i".  ${foo} or
 	# ${1} is OK, but $foo or $1 isn't.
 	# https://bugzilla.sudo.ws/show_bug.cgi?id=765
+
+	# Run prepare
 	sudo --preserve-env=SOURCE_DATE_EPOCH \
 		--preserve-env=BUILDTOOL \
 		--preserve-env=BUILDTOOLVER \
-		-iu builduser bash -c 'cd /startdir; makepkg "$@"' -bash "$@"
+		-iu builduser bash -c 'cd /startdir; makepkg --nobuild "$@"' -bash "$@"
+
+	if in_array "build" "${offline_options[@]}"; then
+		msg "building offline"
+		# Build offline
+		unshare -n -- sudo --preserve-env=SOURCE_DATE_EPOCH \
+			--preserve-env=BUILDTOOL \
+			--preserve-env=BUILDTOOLVER \
+			-iu builduser \
+			bash -c 'cd /startdir; makepkg --noprepare --noextract --nocheck "$@"' -bash "$@"
+	else
+		sudo --preserve-env=SOURCE_DATE_EPOCH \
+			--preserve-env=BUILDTOOL \
+			--preserve-env=BUILDTOOLVER \
+			-iu builduser \
+			bash -c 'cd /startdir; makepkg --noprepare --noextract --nocheck "$@"' -bash "$@"
+	fi
+
+	if in_array "check" "${offline_options[@]}"; then
+		msg "check offline"
+		# Run tests online
+		unshare -n -- sudo --preserve-env=SOURCE_DATE_EPOCH \
+			--preserve-env=BUILDTOOL \
+			--preserve-env=BUILDTOOLVER \
+			-iu builduser \
+			bash -c 'cd /startdir; makepkg --noprepare --noextract --nobuild "$@"' -bash "$@"
+	else
+		sudo --preserve-env=SOURCE_DATE_EPOCH \
+			--preserve-env=BUILDTOOL \
+			--preserve-env=BUILDTOOLVER \
+			-iu builduser \
+			bash -c 'cd /startdir; makepkg --noprepare --noextract --nobuild "$@"' -bash "$@"
+	fi
+
 	ret=$?
 	case $ret in
 		0|14)
@@ -293,7 +335,7 @@ move_products() {
 }
 # }}}
 
-while getopts 'hcur:I:l:nCTD:d:U:x:t:' arg; do
+while getopts 'hcur:I:l:nCTD:o:d:U:x:t:' arg; do
 	case "$arg" in
 		c) clean_first=1 ;;
 		D) bindmounts_ro+=("--bind-ro=$OPTARG") ;;
@@ -304,6 +346,7 @@ while getopts 'hcur:I:l:nCTD:d:U:x:t:' arg; do
 		I) install_pkgs+=("$OPTARG") ;;
 		l) copy="$OPTARG" ;;
 		n) run_namcap=1; makepkg_args+=(--install) ;;
+		o) offline_options+=("$OPTARG") ;;
 		C) run_checkpkg=1 ;;
 		T) temp_chroot=1; copy+="-$$" ;;
 		U) makepkg_user="$OPTARG" ;;