Merge branch 'support-offline-builds' into 'master'

Draft: PoC: build offline support

See merge request archlinux/devtools!207

contrib/completion/bash/devtools.in

@@ -150,7 +150,6 @@ _pkgctl_cmds=(
 	db
 	diff
 	issue
-	license
 	release
 	repo
 	search

src/lib/build/build.sh

@@ -55,6 +55,7 @@ pkgctl_build_usage() {
 		    -o, --offload        Build on a remote server and transfer artifacts afterwards
 		    -c, --clean          Recreate the chroot before building
 		    --inspect WHEN       Spawn an interactive shell to inspect the chroot (never, always, failure)
+		    --offline MODE       Run a part of the build process offline (build, check)
 		    -w, --worker SLOT    Name of the worker slot, useful for concurrent builds (disables automatic names)
 		    --nocheck            Do not run the check() function in the PKGBUILD
 
@@ -198,6 +199,10 @@ pkgctl_build() {
 				EDIT=1
 				shift
 				;;
+			--offline)
+				MAKECHROOT_OPTIONS+=("-o" "$2")
+				shift 2
+				;;
 			-o|--offload)
 				OFFLOAD=1
 				shift
@@ -312,7 +317,7 @@ pkgctl_build() {
 
 	# Update pacman cache for auto-detection
 	if [[ -z ${REPO} ]]; then
-		update_pacman_repo_cache stable
+		update_pacman_repo_cache multilib
 	# Check valid repos if not resolved dynamically
 	elif ! in_array "${REPO}" "${DEVTOOLS_VALID_REPOS[@]}"; then
 		die "Invalid repository target: %s" "${REPO}"

src/lib/db/remove.sh

@@ -51,6 +51,7 @@ pkgctl_db_remove() {
 	local partial=0
 	local confirm=1
 	local dbscripts_options=()
+	local lookup_repo=multilib
 	local pkgname
 
 	# option checking
@@ -105,13 +106,13 @@ pkgctl_db_remove() {
 				update_pacman_repo_cache unstable
 				;;
 			*-staging)
-				update_pacman_repo_cache staging
+				update_pacman_repo_cache multilib-staging
 				;;
 			*-testing)
-				update_pacman_repo_cache testing
+				update_pacman_repo_cache multilib-testing
 				;;
 			*)
-				update_pacman_repo_cache stable
+				update_pacman_repo_cache multilib
 				;;
 		esac
 

src/lib/license/setup.sh

@@ -188,7 +188,6 @@ path = [
     "README.md",
     "keys/**",
     ".SRCINFO",
-    ".gitignore",
     ".nvchecker.toml",
     "*.install",
     "*.sysusers",

src/lib/release.sh

@@ -124,7 +124,7 @@ pkgctl_release() {
 
 	# Update pacman cache for auto-detection
 	if [[ -z ${REPO} ]]; then
-		update_pacman_repo_cache stable
+		update_pacman_repo_cache multilib
 	# Check valid repos if not resolved dynamically
 	elif ! in_array "${REPO}" "${DEVTOOLS_VALID_REPOS[@]}"; then
 		die "Invalid repository target: %s" "${REPO}"

src/lib/util/machine.sh

@@ -1,26 +0,0 @@
-#!/hint/bash
-#
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-[[ -z ${DEVTOOLS_INCLUDE_UTIL_MACHINE_SH:-} ]] || return 0
-DEVTOOLS_INCLUDE_UTIL_MACHINE_SH=1
-
-_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@}
-# shellcheck source=src/lib/common.sh
-source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh
-
-
-set -eo pipefail
-
-machine_get_hardware_name() {
-	uname --machine
-}
-
-machine_has_multilib() {
-	case "$(machine_get_hardware_name)" in
-		x86_64*)
-			return 0
-			;;
-	esac
-	return 1
-}

src/lib/util/pacman.sh

@@ -8,8 +8,6 @@ DEVTOOLS_INCLUDE_UTIL_PACMAN_SH=1
 _DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@}
 # shellcheck source=src/lib/common.sh
 source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh
-# shellcheck source=src/lib/util/machine.sh
-source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/machine.sh
 
 set -e
 
@@ -20,8 +18,7 @@ readonly _DEVTOOLS_MAKEPKG_CONF_DIR=${_DEVTOOLS_LIBRARY_DIR}/makepkg.conf.d
 
 
 update_pacman_repo_cache() {
-	local repo=${1:-stable}
-	repo=$(pacman_resolve_virtual_repo_name "${repo}")
+	local repo=${1:-multilib}
 
 	mkdir -p "${_DEVTOOLS_PACMAN_CACHE_DIR}"
 	msg "Updating pacman database cache"
@@ -35,8 +32,7 @@ update_pacman_repo_cache() {
 
 get_pacman_repo_from_pkgbuild() {
 	local path=${1:-PKGBUILD}
-	local repo=${2:-stable}
-	repo=$(pacman_resolve_virtual_repo_name "${repo}")
+	local repo=${2:-multilib}
 	local -a pkgnames
 
 	# shellcheck source=contrib/makepkg/PKGBUILD.proto
@@ -76,7 +72,6 @@ get_pkgnames_from_repo_pkgbase() {
 
 	# update the pacman repo cache if it doesn't exist yet
 	if [[ ! -d "${_DEVTOOLS_PACMAN_CACHE_DIR}" ]]; then
-		# TODO: universe includes multilib, switch for architecture
 		update_pacman_repo_cache universe
 	fi
 
@@ -96,23 +91,3 @@ get_pkgnames_from_repo_pkgbase() {
 	printf "%s\n" "${pkgnames[@]}"
 	return 0
 }
-
-pacman_resolve_virtual_repo_name() {
-	local repo=$1
-
-	local repo_class=extra
-	if machine_has_multilib; then
-		repo_class=multilib
-	fi
-
-	case "${repo}" in
-		stable)
-			repo=${repo_class}
-			;;
-		testing|staging)
-			repo="${repo_class}-${repo}"
-			;;
-	esac
-
-	printf "%s" "${repo}"
-}

src/lib/valid-tags.sh

@@ -6,7 +6,6 @@
 # shellcheck disable=2034
 DEVTOOLS_VALID_BINARY_ARCHES=(
 	x86_64
-	aarch64
 )
 
 # shellcheck disable=2034

src/makechrootpkg.in

@@ -40,6 +40,8 @@ bindmounts_ro=()
 bindmounts_rw=()
 bindmounts_tmpfs=()
 
+offline_options=()
+
 copy=$USER
 [[ -n ${SUDO_USER:-} ]] && copy=$SUDO_USER
 [[ -z "$copy" || $copy = root ]] && copy=copy
@@ -80,6 +82,7 @@ usage() {
 	echo '                  Useful for maintaining multiple copies'
 	echo "                  Default: $copy"
 	echo '-n                Run namcap on the package'
+	echo '-o                Run given step offline'
 	echo '-C                Run checkpkg on the package'
 	echo '-T                Build in a temporary directory'
 	echo '-U                Run makepkg as a specified user'
@@ -203,6 +206,7 @@ EOF
 		declare -p SOURCE_DATE_EPOCH 2>/dev/null || true
 		declare -p BUILDTOOL 2>/dev/null
 		declare -p BUILDTOOLVER 2>/dev/null
+		declare -p offline_options
 		printf '_chrootbuild "$@" || exit\n'
 
 		if (( run_namcap )); then
@@ -222,14 +226,52 @@ _chrootbuild() {
 	# shellcheck source=/dev/null
 	. /etc/profile
 
+	# for in_array
+	. /usr/share/makepkg/util.sh
+
 	# Beware, there are some stupid arbitrary rules on how you can
 	# use "$" in arguments to commands with "sudo -i".  ${foo} or
 	# ${1} is OK, but $foo or $1 isn't.
 	# https://bugzilla.sudo.ws/show_bug.cgi?id=765
+
+	# Run prepare
 	sudo --preserve-env=SOURCE_DATE_EPOCH \
 		--preserve-env=BUILDTOOL \
 		--preserve-env=BUILDTOOLVER \
-		-iu builduser bash -c 'cd /startdir; makepkg "$@"' -bash "$@"
+		-iu builduser bash -c 'cd /startdir; makepkg --nobuild "$@"' -bash "$@"
+
+	if in_array "build" "${offline_options[@]}"; then
+		msg "building offline"
+		# Build offline
+		unshare -n -- sudo --preserve-env=SOURCE_DATE_EPOCH \
+			--preserve-env=BUILDTOOL \
+			--preserve-env=BUILDTOOLVER \
+			-iu builduser \
+			bash -c 'cd /startdir; makepkg --noprepare --noextract --nocheck "$@"' -bash "$@"
+	else
+		sudo --preserve-env=SOURCE_DATE_EPOCH \
+			--preserve-env=BUILDTOOL \
+			--preserve-env=BUILDTOOLVER \
+			-iu builduser \
+			bash -c 'cd /startdir; makepkg --noprepare --noextract --nocheck "$@"' -bash "$@"
+	fi
+
+	if in_array "check" "${offline_options[@]}"; then
+		msg "check offline"
+		# Run tests online
+		unshare -n -- sudo --preserve-env=SOURCE_DATE_EPOCH \
+			--preserve-env=BUILDTOOL \
+			--preserve-env=BUILDTOOLVER \
+			-iu builduser \
+			bash -c 'cd /startdir; makepkg --noprepare --noextract --nobuild "$@"' -bash "$@"
+	else
+		sudo --preserve-env=SOURCE_DATE_EPOCH \
+			--preserve-env=BUILDTOOL \
+			--preserve-env=BUILDTOOLVER \
+			-iu builduser \
+			bash -c 'cd /startdir; makepkg --noprepare --noextract --nobuild "$@"' -bash "$@"
+	fi
+
 	ret=$?
 	case $ret in
 		0|14)
@@ -293,7 +335,7 @@ move_products() {
 }
 # }}}
 
-while getopts 'hcur:I:l:nCTD:d:U:x:t:' arg; do
+while getopts 'hcur:I:l:nCTD:o:d:U:x:t:' arg; do
 	case "$arg" in
 		c) clean_first=1 ;;
 		D) bindmounts_ro+=("--bind-ro=$OPTARG") ;;
@@ -304,6 +346,7 @@ while getopts 'hcur:I:l:nCTD:d:U:x:t:' arg; do
 		I) install_pkgs+=("$OPTARG") ;;
 		l) copy="$OPTARG" ;;
 		n) run_namcap=1; makepkg_args+=(--install) ;;
+		o) offline_options+=("$OPTARG") ;;
 		C) run_checkpkg=1 ;;
 		T) temp_chroot=1; copy+="-$$" ;;
 		U) makepkg_user="$OPTARG" ;;