Merge branch 'support-offline-builds' into 'master'

Draft: PoC: build offline support

See merge request archlinux/devtools!207

doc/man/offload-build.1.asciidoc

@@ -14,7 +14,7 @@ Description
 
 Build a PKGBUILD on a remote server using makechrootpkg. Requires a remote user
 that can run archbuild in a non-interactive manner, e.g. must be able to
-elevate permissions using passwordless run0.
+elevate permissions using passwordless sudo.
 
 Options
 -------

doc/man/pkgctl-auth.1.asciidoc

@@ -3,7 +3,7 @@ pkgctl-auth(1)
 
 Name
 ----
-pkgctl-auth - Authenticate with services like GitLab.
+pkgctl-auth - Authenticate with serivces like GitLab.
 
 Synopsis
 --------

src/lib/archroot.sh

@@ -15,11 +15,7 @@ check_root() {
 	local orig_argv=("$@")
 
 	(( EUID == 0 )) && return
-	if type -P run0 >/dev/null; then
-		keepenv=",$keepenv"
-		command="run0 ${keepenv//,/ --setenv=}"
-		exec ${command} -- "${orig_argv[@]}"
-	elif type -P sudo >/dev/null; then
+	if type -P sudo >/dev/null; then
 		exec sudo --preserve-env="${keepenv}" -- "${orig_argv[@]}"
 	else
 		exec su root -c "$(printf ' %q' "${orig_argv[@]}")"

src/lib/build/build.sh

@@ -55,6 +55,7 @@ pkgctl_build_usage() {
 		    -o, --offload        Build on a remote server and transfer artifacts afterwards
 		    -c, --clean          Recreate the chroot before building
 		    --inspect WHEN       Spawn an interactive shell to inspect the chroot (never, always, failure)
+		    --offline MODE       Run a part of the build process offline (build, check)
 		    -w, --worker SLOT    Name of the worker slot, useful for concurrent builds (disables automatic names)
 		    --nocheck            Do not run the check() function in the PKGBUILD
 
@@ -198,6 +199,10 @@ pkgctl_build() {
 				EDIT=1
 				shift
 				;;
+			--offline)
+				MAKECHROOT_OPTIONS+=("-o" "$2")
+				shift 2
+				;;
 			-o|--offload)
 				OFFLOAD=1
 				shift

src/makechrootpkg.in

@@ -40,6 +40,8 @@ bindmounts_ro=()
 bindmounts_rw=()
 bindmounts_tmpfs=()
 
+offline_options=()
+
 copy=$USER
 [[ -n ${SUDO_USER:-} ]] && copy=$SUDO_USER
 [[ -z "$copy" || $copy = root ]] && copy=copy
@@ -80,6 +82,7 @@ usage() {
 	echo '                  Useful for maintaining multiple copies'
 	echo "                  Default: $copy"
 	echo '-n                Run namcap on the package'
+	echo '-o                Run given step offline'
 	echo '-C                Run checkpkg on the package'
 	echo '-T                Build in a temporary directory'
 	echo '-U                Run makepkg as a specified user'
@@ -185,18 +188,10 @@ prepare_chroot() {
 		echo "$x" >>"$copydir/etc/makepkg.conf"
 	done
 
-	# TODO(gromit): check if this rule is sane
-	# TODO(gromit): this will require a full container
-	cat > "$copydir/etc/polkit-1/rules.d/10-systemd-nopasswd.rules" <<EOF
-polkit.addRule(function(action, subject) {
-    if (action.id == "org.freedesktop.systemd1.manage-units") {
-        if (subject.isInGroup("wheel")) {
-            return polkit.Result.YES;
-        }
-    }
-});
+	cat > "$copydir/etc/sudoers.d/builduser-pacman" <<EOF
+builduser ALL = NOPASSWD: /usr/bin/pacman
 EOF
-	chmod 440 "$copydir/etc/polkit-1/rules.d/10-systemd-nopasswd.rules"
+	chmod 440 "$copydir/etc/sudoers.d/builduser-pacman"
 
 	cat > "$copydir/etc/gitconfig" <<EOF
 [safe]
@@ -211,6 +206,7 @@ EOF
 		declare -p SOURCE_DATE_EPOCH 2>/dev/null || true
 		declare -p BUILDTOOL 2>/dev/null
 		declare -p BUILDTOOLVER 2>/dev/null
+		declare -p offline_options
 		printf '_chrootbuild "$@" || exit\n'
 
 		if (( run_namcap )); then
@@ -230,14 +226,55 @@ _chrootbuild() {
 	# shellcheck source=/dev/null
 	. /etc/profile
 
-	run0 --setenv=SOURCE_DATE_EPOCH \
-		 --setenv=BUILDTOOL \
-		 --setenv=BUILDTOOLVER \
-		 --via-shell --chdir='~' \
-		 --user=builduser -- bash -c 'cd /startdir; makepkg "$@"' -bash "$@"
+	# for in_array
+	. /usr/share/makepkg/util.sh
+
+	# Beware, there are some stupid arbitrary rules on how you can
+	# use "$" in arguments to commands with "sudo -i".  ${foo} or
+	# ${1} is OK, but $foo or $1 isn't.
+	# https://bugzilla.sudo.ws/show_bug.cgi?id=765
+
+	# Run prepare
+	sudo --preserve-env=SOURCE_DATE_EPOCH \
+		--preserve-env=BUILDTOOL \
+		--preserve-env=BUILDTOOLVER \
+		-iu builduser bash -c 'cd /startdir; makepkg --nobuild "$@"' -bash "$@"
+
+	if in_array "build" "${offline_options[@]}"; then
+		msg "building offline"
+		# Build offline
+		unshare -n -- sudo --preserve-env=SOURCE_DATE_EPOCH \
+			--preserve-env=BUILDTOOL \
+			--preserve-env=BUILDTOOLVER \
+			-iu builduser \
+			bash -c 'cd /startdir; makepkg --noprepare --noextract --nocheck "$@"' -bash "$@"
+	else
+		sudo --preserve-env=SOURCE_DATE_EPOCH \
+			--preserve-env=BUILDTOOL \
+			--preserve-env=BUILDTOOLVER \
+			-iu builduser \
+			bash -c 'cd /startdir; makepkg --noprepare --noextract --nocheck "$@"' -bash "$@"
+	fi
+
+	if in_array "check" "${offline_options[@]}"; then
+		msg "check offline"
+		# Run tests online
+		unshare -n -- sudo --preserve-env=SOURCE_DATE_EPOCH \
+			--preserve-env=BUILDTOOL \
+			--preserve-env=BUILDTOOLVER \
+			-iu builduser \
+			bash -c 'cd /startdir; makepkg --noprepare --noextract --nobuild "$@"' -bash "$@"
+	else
+		sudo --preserve-env=SOURCE_DATE_EPOCH \
+			--preserve-env=BUILDTOOL \
+			--preserve-env=BUILDTOOLVER \
+			-iu builduser \
+			bash -c 'cd /startdir; makepkg --noprepare --noextract --nobuild "$@"' -bash "$@"
+	fi
+
 	ret=$?
 	case $ret in
-		0)
+		0|14)
 			return 0;;
 		*)
 			return $ret;;
@@ -248,7 +285,7 @@ _chrootnamcap() {
 	pacman -S --needed --noconfirm namcap
 	for pkgfile in /startdir/PKGBUILD /pkgdest/*; do
 		echo "Checking ${pkgfile##*/}"
-		run0 --user=builduser -- namcap "$pkgfile" 2>&1 | tee "/logdest/${pkgfile##*/}-namcap.log"
+		sudo -u builduser namcap "$pkgfile" 2>&1 | tee "/logdest/${pkgfile##*/}-namcap.log"
 	done
 }
 
@@ -257,12 +294,8 @@ download_sources() {
 	chown "$makepkg_user:" "$WORKDIR"
 
 	# Ensure sources are downloaded
-	run0 --user="$makepkg_user" \
-		--setenv=GNUPGHOME \
-		--setenv=SSH_AUTH_SOCK \
-		--setenv=SRCDEST="$SRCDEST" \
-		--setenv=BUILDDIR="$WORKDIR" \
-		--chdir=. -- \
+	sudo -u "$makepkg_user" --preserve-env=GNUPGHOME,SSH_AUTH_SOCK \
+		env SRCDEST="$SRCDEST" BUILDDIR="$WORKDIR" \
 		makepkg --config="$copydir/etc/makepkg.conf" --verifysource -o "${verifysource_args[@]}" ||
 		die "Could not download sources."
 }
@@ -302,7 +335,7 @@ move_products() {
 }
 # }}}
 
-while getopts 'hcur:I:l:nCTD:d:U:x:t:' arg; do
+while getopts 'hcur:I:l:nCTD:o:d:U:x:t:' arg; do
 	case "$arg" in
 		c) clean_first=1 ;;
 		D) bindmounts_ro+=("--bind-ro=$OPTARG") ;;
@@ -313,6 +346,7 @@ while getopts 'hcur:I:l:nCTD:d:U:x:t:' arg; do
 		I) install_pkgs+=("$OPTARG") ;;
 		l) copy="$OPTARG" ;;
 		n) run_namcap=1; makepkg_args+=(--install) ;;
+		o) offline_options+=("$OPTARG") ;;
 		C) run_checkpkg=1 ;;
 		T) temp_chroot=1; copy+="-$$" ;;
 		U) makepkg_user="$OPTARG" ;;
@@ -409,7 +443,7 @@ if arch-nspawn "$copydir" \
 	"${nspawn_build_args[@]}" \
 	/chrootbuild "${makepkg_args[@]}"
 then
-	mapfile -t pkgnames < <(run0 --user="$makepkg_user" -- bash -c 'source PKGBUILD; printf "%s\n" "${pkgname[@]}"')
+	mapfile -t pkgnames < <(sudo -u "$makepkg_user" bash -c 'source PKGBUILD; printf "%s\n" "${pkgname[@]}"')
 	move_products
 else
 	(( ret += 1 ))
@@ -462,7 +496,7 @@ else
 		done
 
 		msg2 "Checking packages"
-		run0 --user="$makepkg_user" -- checkpkg --rmdir --warn --makepkg-config "$copydir/etc/makepkg.conf" "${remotepkgs[@]/#file:\/\//}"
+		sudo -u "$makepkg_user" checkpkg --rmdir --warn --makepkg-config "$copydir/etc/makepkg.conf" "${remotepkgs[@]/#file:\/\//}"
 	fi
 	true
 fi

src/makerepropkg.in

@@ -192,7 +192,7 @@ for p in "$@"; do
         pkgfile=${pkgfile_remote#file://}
         if [[ ! -f ${pkgfile} ]]; then
             msg "Downloading package '%s' into pacman's cache" "${pkgfile}"
-            run0 -- pacman -Swdd --noconfirm --logfile /dev/null "${p}" || exit 1
+            sudo pacman -Swdd --noconfirm --logfile /dev/null "${p}" || exit 1
             pkgfile_remote=$(pacman -Sddp "${p}" 2>/dev/null)
             pkgfile="${pkgfile_remote#file://}"
         fi