Merge branch '211-pkgctl-repo-update' into 'master'

feat: introduce the pkgctl repo pull command

Closes #211

See merge request archlinux/devtools!235

contrib/completion/bash/devtools.in

@@ -150,7 +150,6 @@ _pkgctl_cmds=(
 	db
 	diff
 	issue
-	license
 	release
 	repo
 	search
@@ -306,6 +305,7 @@ _pkgctl_repo_cmds=(
 	clone
 	configure
 	create
+	pull
 	switch
 	web
 )
@@ -335,6 +335,19 @@ _pkgctl_repo_clean_args=(
 )
 _pkgctl_repo_clean_opts() { _filedir -d; }
 
+_pkgctl_repo_pull_args=(
+	--discard-changes
+	--show-diff
+	--quiet
+	--autostash
+	-f --force
+	-j --jobs
+	-h --help
+)
+_pkgctl_repo_pull_args__jobs_opts() { :; }
+_pkgctl_repo_pull_args_j_opts() { _pkgctl_repo_pull_args__jobs_opts; }
+_pkgctl_repo_pull_opts() { _filedir -d; }
+
 _pkgctl_repo_configure_args=(
 	--protocol
 	-j --jobs

contrib/completion/zsh/_devtools.in

@@ -228,6 +228,7 @@ _pkgctl_repo_cmds=(
 	"clone[Clone a package repository]"
 	"configure[Configure a clone according to distro specs]"
 	"create[Create a new GitLab package repository]"
+	"pull[Update package repositories from their git remote]"
 	"switch[Switch a package repository to a specified version]"
 	"web[Open the packaging repository's website]"
 )
@@ -256,6 +257,17 @@ _pkgctl_repo_clone_args=(
 	'*:packages:_devtools_completions_all_packages'
 )
 
+_pkgctl_repo_pull_args=(
+	'--discard-changes[Discard changes if index or working tree is dirty]'
+	'--quiet[Disable printing longer terminal output]'
+	'--show-diff[Always enable showing the diff]'
+	'--autostash[Stash before pulling and unstash afterwards]'
+	'(-f --force)'{-f,--force}'[Alias for --discard-changes]'
+	'(-j --jobs)'{-j,--jobs}'[Run up to N jobs in parallel (default: number of processing units)]:jobs:'
+	'(-h --help)'{-h,--help}'[Display usage]'
+	'*:git_dir:_files -/'
+)
+
 _pkgctl_repo_configure_args=(
 	'--protocol[Configure remote url to use https]:proto:(https)'
 	'(-j --jobs)'{-j,--jobs}'[Run up to N jobs in parallel (default: number of processing units)]:jobs:'

doc/man/offload-build.1.asciidoc

@@ -14,7 +14,7 @@ Description
 
 Build a PKGBUILD on a remote server using makechrootpkg. Requires a remote user
 that can run archbuild in a non-interactive manner, e.g. must be able to
-elevate permissions using passwordless run0.
+elevate permissions using passwordless sudo.
 
 Options
 -------

doc/man/pkgctl-auth.1.asciidoc

@@ -3,7 +3,7 @@ pkgctl-auth(1)
 
 Name
 ----
-pkgctl-auth - Authenticate with services like GitLab.
+pkgctl-auth - Authenticate with serivces like GitLab.
 
 Synopsis
 --------

doc/man/pkgctl-repo-pull.1.asciidoc

@@ -0,0 +1,50 @@
+pkgctl-repo-pull(1)
+===================
+
+Name
+----
+pkgctl-repo-pull - Pull in git changes
+
+Synopsis
+--------
+pkgctl repo pull [OPTIONS] [PATH...]
+
+Description
+-----------
+
+Update package repositories from their git remotes.
+
+If only a single package is given the command also automatically shows a diff
+of what has changed since the last time the repository was updated.
+
+Options
+-------
+
+*--discard-changes*::
+	Discard changes if the index or working tree was modified. Otherwise abort
+	if anything was modified.
+
+*--autostash*::
+	Stash before pulling and unstash afterwards
+
+*--quiet*::
+	Do not show any longer terminal output like diffs.
+
+*--show-diff*::
+	Show what has changed since the last time the repository was updated. Is
+	automatically set when only one 'PATH' is given.
+
+*-j, --jobs* 'N'::
+	Run up to N jobs in parallel. By default the number of jobs is equal to the
+	number of available processing units. For sequential processing this option
+	needs to be passed with 1.
+
+*-h, --help*::
+	Show a help text
+
+See Also
+--------
+
+git-pull(1)
+
+include::include/footer.asciidoc[]

doc/man/pkgctl-repo.1.asciidoc

@@ -44,6 +44,9 @@ pkgctl repo configure::
 pkgctl repo create::
 	Create a new GitLab package repository
 
+pkgctl repo pull::
+	Update package repositories from their git remotes
+
 pkgctl repo switch::
 	Switch a package repository to a specified version
 
@@ -57,6 +60,7 @@ pkgctl-repo-clean(1)
 pkgctl-repo-clone(1)
 pkgctl-repo-configure(1)
 pkgctl-repo-create(1)
+pkgctl-repo-pull(1)
 pkgctl-repo-switch(1)
 pkgctl-repo-web(1)
 

src/lib/archroot.sh

@@ -15,11 +15,7 @@ check_root() {
 	local orig_argv=("$@")
 
 	(( EUID == 0 )) && return
-	if type -P run0 >/dev/null; then
-		keepenv=",$keepenv"
-		command="run0 ${keepenv//,/ --setenv=}"
-		exec ${command} -- "${orig_argv[@]}"
-	elif type -P sudo >/dev/null; then
+	if type -P sudo >/dev/null; then
 		exec sudo --preserve-env="${keepenv}" -- "${orig_argv[@]}"
 	else
 		exec su root -c "$(printf ' %q' "${orig_argv[@]}")"

src/lib/license/setup.sh

@@ -188,7 +188,6 @@ path = [
     "README.md",
     "keys/**",
     ".SRCINFO",
-    ".gitignore",
     ".nvchecker.toml",
     "*.install",
     "*.sysusers",

src/lib/repo.sh

@@ -31,6 +31,7 @@ pkgctl_repo_usage() {
 		    clone          Clone a package repository
 		    configure      Configure a clone according to distro specs
 		    create         Create a new GitLab package repository
+		    pull           Update package repositories from their git remote
 		    switch         Switch a package repository to a specified version
 		    web            Open the packaging repository's website
 
@@ -40,6 +41,7 @@ pkgctl_repo_usage() {
 		EXAMPLES
 		    $ ${COMMAND} clean --interactive *
 		    $ ${COMMAND} clone libfoo linux libbar
+		    $ ${COMMAND} pull libfoo linux libbar
 		    $ ${COMMAND} clone --maintainer mynickname
 		    $ ${COMMAND} configure *
 		    $ ${COMMAND} create libfoo
@@ -101,6 +103,14 @@ pkgctl_repo() {
 				pkgctl_repo_switch "$@"
 				exit 0
 				;;
+			pull)
+				_DEVTOOLS_COMMAND+=" $1"
+				shift
+				# shellcheck source=src/lib/repo/pull.sh
+				source "${_DEVTOOLS_LIBRARY_DIR}"/lib/repo/pull.sh
+				pkgctl_repo_pull "$@"
+				exit 0
+				;;
 			web)
 				_DEVTOOLS_COMMAND+=" $1"
 				shift

src/lib/repo/pull.sh

@@ -0,0 +1,195 @@
+#!/bin/bash
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+[[ -z ${DEVTOOLS_INCLUDE_REPO_PULL_SH:-} ]] || return 0
+DEVTOOLS_INCLUDE_REPO_PULL_SH=1
+
+_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@}
+# shellcheck source=src/lib/common.sh
+source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh
+# shellcheck source=src/lib/util/git.sh
+source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/git.sh
+
+source /usr/share/makepkg/util/message.sh
+
+set -eo pipefail
+
+
+pkgctl_repo_pull_usage() {
+	local -r COMMAND=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}}
+	cat <<- _EOF_
+		Usage: ${COMMAND} [OPTIONS] [PKGBASE]...
+
+		Update package repositories from their git remotes.
+
+		OPTIONS
+		    --discard-changes   Discard changes if index or working tree is dirty
+		    --show-diff         Always enable showing the diff
+		    --autostash         Stash before pulling and unstash afterwards
+		    -j, --jobs N        Run up to N jobs in parallel (default: $(nproc))
+		    --quiet             Disable printing longer terminal output
+		    -h, --help          Show this help text
+
+		EXAMPLES
+		    $ ${COMMAND} gopass gopass-jsonapi
+_EOF_
+}
+
+pkgctl_repo_pull() {
+	# options
+	local paths path pkgbase branch jobs remote
+	jobs=$(nproc)
+	local command=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}}
+	local git_rebase_options=()
+	local SHOW_DIFF=0
+	local QUIET=0
+	local DISCARD_CHANGES=0
+	local AUTOSTASH=0
+
+	while (( $# )); do
+		case $1 in
+			-h|--help)
+				pkgctl_repo_pull_usage
+				exit 0
+				;;
+			--discard-changes)
+				DISCARD_CHANGES=1
+				shift
+				;;
+			--show-diff)
+				SHOW_DIFF=1
+				shift
+				;;
+			--quiet)
+				QUIET=1
+				shift
+				;;
+			--autostash)
+				AUTOSTASH=1
+				git_rebase_options+=(--autostash)
+				shift
+				;;
+			-j|--jobs)
+				(( $# <= 1 )) && die "missing argument for %s" "$1"
+				jobs=$2
+				shift 2
+				;;
+			--)
+				shift
+				break
+				;;
+			-*)
+				die "invalid argument: %s" "$1"
+				;;
+			*)
+				paths=("$@")
+				break
+				;;
+		esac
+	done
+
+	# check if invoked without any path from within a packaging repo
+	if (( ${#paths[@]} == 0 )); then
+		paths=(".")
+	fi
+
+	# check if we are only working on one repo and enable
+	if (( ${#paths[@]} == 1 )) && (( ! QUIET )); then
+		SHOW_DIFF=1
+	fi
+
+	# parallelization
+	if [[ ${jobs} != 1 ]] && (( ${#paths[@]} > 1 )); then
+		if [[ -n ${BOLD} ]]; then
+			export DEVTOOLS_COLOR=always
+		fi
+
+		# warm up ssh connection as it may require user input (key unlock, hostkey verification etc)
+		git_warmup_ssh_connection
+
+		# disable diffs if not enabled explicitly
+		if (( ! SHOW_DIFF )); then
+			command+=" --quiet"
+		fi
+
+		if (( DISCARD_CHANGES )); then
+			command+=" --discard-changes"
+		fi
+
+		if (( AUTOSTASH )); then
+			command+=" --autostash"
+		fi
+
+		if ! parallel --bar --jobs "${jobs}" "${command}" ::: "${paths[@]}"; then
+			die 'Failed to pull some packages, please check the previous output'
+		fi
+		exit 0
+	fi
+
+	for path in "${paths[@]}"; do
+		# skip paths that are not directories
+		if [[ ! -d "${path}" ]]; then
+			continue
+		fi
+
+		if [[ ! -f "${path}/PKGBUILD" ]]; then
+			msg_error "  Not a package repository: ${path}"
+			continue
+		fi
+
+		if [[ ! -d "${path}/.git" ]]; then
+			msg_error "  Not a Git repository: ${path}"
+			continue
+		fi
+
+		pkgbase=$(basename "$(realpath "${path}")")
+		pkgbase=${pkgbase%.git}
+		msg "Updating ${pkgbase}"
+
+		branch=$(git -C "${path}" symbolic-ref --quiet --short HEAD)
+		if [[ ${branch} != main ]]; then
+			msg_warn "  Current branch is ${branch}, not updating from canonical upstream: ${pkgbase}"
+		fi
+
+		if ! git -C "${path}" diff-files --quiet && (( ! AUTOSTASH )) && (( ! DISCARD_CHANGES )); then
+			msg_error "  Index contains unstaged changes, please stash them or pass --autostash before pulling: ${pkgbase}"
+			continue
+		fi
+
+		if ! git -C "${path}" diff-index --cached --quiet HEAD && (( ! AUTOSTASH )) && (( ! DISCARD_CHANGES )); then
+			msg_error "  Index contains uncommited changes, please commit or stash them before pulling: ${pkgbase}"
+			continue
+		fi
+
+		remote=$(git -C "${path}" config "branch.${branch}.remote")
+		if [[ -z "${remote}" ]]; then
+			msg_error "  No upstream tracking branch configured: ${pkgbase}"
+			continue
+		fi
+
+		if ! git -C "${path}" fetch --quiet "${remote}"; then
+			msg_error "  Error while fetching: ${pkgbase}"
+			continue
+		fi
+
+		if [[ $(git -C "${path}" rev-parse HEAD) == $(git -C "${path}" rev-parse FETCH_HEAD) ]]; then
+			msg2 "Repo is up to date, nothing to do"
+			continue
+		fi
+
+		# discard any local modifications
+		if (( DISCARD_CHANGES )); then
+			git -C "${path}" restore --staged --worktree -- .
+		fi
+
+		if (( SHOW_DIFF )) && (( ! QUIET )); then
+			git -C "${path}" --no-pager diff --color --patch-with-stat HEAD..FETCH_HEAD
+		fi
+
+		if ! git -C "${path}" rebase --quiet "${git_rebase_options[@]}" "${remote}/${branch}"; then
+			msg_error "  Error while pulling in the changes for ${pkgbase}"
+			exit 1
+		fi
+	done
+}

src/makechrootpkg.in

@@ -185,18 +185,10 @@ prepare_chroot() {
 		echo "$x" >>"$copydir/etc/makepkg.conf"
 	done
 
-	# TODO(gromit): check if this rule is sane
-	# TODO(gromit): this will require a full container
-	cat > "$copydir/etc/polkit-1/rules.d/10-systemd-nopasswd.rules" <<EOF
-polkit.addRule(function(action, subject) {
-    if (action.id == "org.freedesktop.systemd1.manage-units") {
-        if (subject.isInGroup("wheel")) {
-            return polkit.Result.YES;
-        }
-    }
-});
+	cat > "$copydir/etc/sudoers.d/builduser-pacman" <<EOF
+builduser ALL = NOPASSWD: /usr/bin/pacman
 EOF
-	chmod 440 "$copydir/etc/polkit-1/rules.d/10-systemd-nopasswd.rules"
+	chmod 440 "$copydir/etc/sudoers.d/builduser-pacman"
 
 	cat > "$copydir/etc/gitconfig" <<EOF
 [safe]
@@ -230,14 +222,17 @@ _chrootbuild() {
 	# shellcheck source=/dev/null
 	. /etc/profile
 
-	run0 --setenv=SOURCE_DATE_EPOCH \
-		 --setenv=BUILDTOOL \
-		 --setenv=BUILDTOOLVER \
-		 --via-shell --chdir='~' \
-		 --user=builduser -- bash -c 'cd /startdir; makepkg "$@"' -bash "$@"
+	# Beware, there are some stupid arbitrary rules on how you can
+	# use "$" in arguments to commands with "sudo -i".  ${foo} or
+	# ${1} is OK, but $foo or $1 isn't.
+	# https://bugzilla.sudo.ws/show_bug.cgi?id=765
+	sudo --preserve-env=SOURCE_DATE_EPOCH \
+		--preserve-env=BUILDTOOL \
+		--preserve-env=BUILDTOOLVER \
+		-iu builduser bash -c 'cd /startdir; makepkg "$@"' -bash "$@"
 	ret=$?
 	case $ret in
-		0)
+		0|14)
 			return 0;;
 		*)
 			return $ret;;
@@ -248,7 +243,7 @@ _chrootnamcap() {
 	pacman -S --needed --noconfirm namcap
 	for pkgfile in /startdir/PKGBUILD /pkgdest/*; do
 		echo "Checking ${pkgfile##*/}"
-		run0 --user=builduser -- namcap "$pkgfile" 2>&1 | tee "/logdest/${pkgfile##*/}-namcap.log"
+		sudo -u builduser namcap "$pkgfile" 2>&1 | tee "/logdest/${pkgfile##*/}-namcap.log"
 	done
 }
 
@@ -257,12 +252,8 @@ download_sources() {
 	chown "$makepkg_user:" "$WORKDIR"
 
 	# Ensure sources are downloaded
-	run0 --user="$makepkg_user" \
-		--setenv=GNUPGHOME \
-		--setenv=SSH_AUTH_SOCK \
-		--setenv=SRCDEST="$SRCDEST" \
-		--setenv=BUILDDIR="$WORKDIR" \
-		--chdir=. -- \
+	sudo -u "$makepkg_user" --preserve-env=GNUPGHOME,SSH_AUTH_SOCK \
+		env SRCDEST="$SRCDEST" BUILDDIR="$WORKDIR" \
 		makepkg --config="$copydir/etc/makepkg.conf" --verifysource -o "${verifysource_args[@]}" ||
 		die "Could not download sources."
 }
@@ -409,7 +400,7 @@ if arch-nspawn "$copydir" \
 	"${nspawn_build_args[@]}" \
 	/chrootbuild "${makepkg_args[@]}"
 then
-	mapfile -t pkgnames < <(run0 --user="$makepkg_user" -- bash -c 'source PKGBUILD; printf "%s\n" "${pkgname[@]}"')
+	mapfile -t pkgnames < <(sudo -u "$makepkg_user" bash -c 'source PKGBUILD; printf "%s\n" "${pkgname[@]}"')
 	move_products
 else
 	(( ret += 1 ))
@@ -462,7 +453,7 @@ else
 		done
 
 		msg2 "Checking packages"
-		run0 --user="$makepkg_user" -- checkpkg --rmdir --warn --makepkg-config "$copydir/etc/makepkg.conf" "${remotepkgs[@]/#file:\/\//}"
+		sudo -u "$makepkg_user" checkpkg --rmdir --warn --makepkg-config "$copydir/etc/makepkg.conf" "${remotepkgs[@]/#file:\/\//}"
 	fi
 	true
 fi

src/makerepropkg.in

@@ -192,7 +192,7 @@ for p in "$@"; do
         pkgfile=${pkgfile_remote#file://}
         if [[ ! -f ${pkgfile} ]]; then
             msg "Downloading package '%s' into pacman's cache" "${pkgfile}"
-            run0 -- pacman -Swdd --noconfirm --logfile /dev/null "${p}" || exit 1
+            sudo pacman -Swdd --noconfirm --logfile /dev/null "${p}" || exit 1
             pkgfile_remote=$(pacman -Sddp "${p}" 2>/dev/null)
             pkgfile="${pkgfile_remote#file://}"
         fi