Compare commits

..

1 Commits

Author SHA1 Message Date
Levente Polyak
c75a9fc47a feat(build): Invoke pkgctl offload server subcommand over ssh
Use a client-server script invocation instead of piping complex shell
commands as pure strings to the remote. Now we call a server use only
component on the remote server to execute the desired build.

This change allows for properly developing the remote command without
golfind around shell strings and while having appropriate completion,
syntax highlight and LSP support. As a side-effect we are now not
depending on a POSIX compliant login shell anymore, as we simply invike
a pkgctl subcommand schema.

Component: pkgctl build
Signed-off-by: Levente Polyak <anthraxx@archlinux.org>
2025-01-23 21:00:36 +01:00
32 changed files with 266 additions and 1079 deletions

View File

@@ -1,6 +1,6 @@
SHELL=/bin/bash -o pipefail SHELL=/bin/bash -o pipefail
V=1.4.0 V=1.3.1
BUILDTOOLVER ?= $(V) BUILDTOOLVER ?= $(V)
PREFIX = /usr/local PREFIX = /usr/local
@@ -19,7 +19,6 @@ PACMAN_CONFIGS=$(wildcard config/pacman/*)
GIT_CONFIGS = $(wildcard config/git/*) GIT_CONFIGS = $(wildcard config/git/*)
SETARCH_ALIASES = $(wildcard config/setarch-aliases.d/*) SETARCH_ALIASES = $(wildcard config/setarch-aliases.d/*)
MANS = $(addprefix $(BUILDDIR)/,$(patsubst %.asciidoc,%,$(wildcard doc/man/*.asciidoc))) MANS = $(addprefix $(BUILDDIR)/,$(patsubst %.asciidoc,%,$(wildcard doc/man/*.asciidoc)))
DATA_FILES = $(wildcard data/*)
COMMITPKG_LINKS = \ COMMITPKG_LINKS = \
core-testingpkg \ core-testingpkg \
@@ -60,7 +59,7 @@ BATS_ARGS ?= --jobs $(JOBS) $(BATS_EXTRA_ARGS) --verbose-run
COVERAGE_DIR ?= $(BUILDDIR)/coverage COVERAGE_DIR ?= $(BUILDDIR)/coverage
all: binprogs library conf completion man data all: binprogs library conf completion man
binprogs: $(BINPROGS) binprogs: $(BINPROGS)
library: $(LIBRARY) library: $(LIBRARY)
completion: $(COMPLETIONS) completion: $(COMPLETIONS)
@@ -107,16 +106,12 @@ $(BUILDDIR)/doc/man/%: doc/man/%.asciidoc doc/man/include/footer.asciidoc
conf: conf:
@install -d $(BUILDDIR)/makepkg.conf.d @install -d $(BUILDDIR)/makepkg.conf.d
@cp -ra $(MAKEPKG_CONFIGS) $(BUILDDIR)/makepkg.conf.d @cp -a $(MAKEPKG_CONFIGS) $(BUILDDIR)/makepkg.conf.d
@install -d $(BUILDDIR)/pacman.conf.d @install -d $(BUILDDIR)/pacman.conf.d
@cp -a $(PACMAN_CONFIGS) $(BUILDDIR)/pacman.conf.d @cp -a $(PACMAN_CONFIGS) $(BUILDDIR)/pacman.conf.d
@install -d $(BUILDDIR)/git.conf.d @install -d $(BUILDDIR)/git.conf.d
@cp -a $(GIT_CONFIGS) $(BUILDDIR)/git.conf.d @cp -a $(GIT_CONFIGS) $(BUILDDIR)/git.conf.d
data:
@install -d $(BUILDDIR)/data
@cp -ra $(DATA_FILES) $(BUILDDIR)/data
clean: clean:
rm -rf $(BUILDDIR) rm -rf $(BUILDDIR)
@@ -127,11 +122,9 @@ install: all
install -dm0755 $(DESTDIR)$(DATADIR)/pacman.conf.d install -dm0755 $(DESTDIR)$(DATADIR)/pacman.conf.d
install -m0755 ${BINPROGS} $(DESTDIR)$(PREFIX)/bin install -m0755 ${BINPROGS} $(DESTDIR)$(PREFIX)/bin
install -dm0755 $(DESTDIR)$(DATADIR)/lib install -dm0755 $(DESTDIR)$(DATADIR)/lib
install -dm0755 $(DESTDIR)$(DATADIR)/data
cp -ra $(BUILDDIR)/lib/* $(DESTDIR)$(DATADIR)/lib cp -ra $(BUILDDIR)/lib/* $(DESTDIR)$(DATADIR)/lib
cp -a $(BUILDDIR)/git.conf.d -t $(DESTDIR)$(DATADIR) cp -a $(BUILDDIR)/git.conf.d -t $(DESTDIR)$(DATADIR)
cp -ra $(BUILDDIR)/makepkg.conf.d -t $(DESTDIR)$(DATADIR) for conf in $(notdir $(MAKEPKG_CONFIGS)); do install -Dm0644 $(BUILDDIR)/makepkg.conf.d/$$conf $(DESTDIR)$(DATADIR)/makepkg.conf.d/$${conf##*/}; done
cp -ra $(BUILDDIR)/data -t $(DESTDIR)$(DATADIR)
for conf in $(notdir $(PACMAN_CONFIGS)); do install -Dm0644 $(BUILDDIR)/pacman.conf.d/$$conf $(DESTDIR)$(DATADIR)/pacman.conf.d/$${conf##*/}; done for conf in $(notdir $(PACMAN_CONFIGS)); do install -Dm0644 $(BUILDDIR)/pacman.conf.d/$$conf $(DESTDIR)$(DATADIR)/pacman.conf.d/$${conf##*/}; done
for a in ${SETARCH_ALIASES}; do install -m0644 $$a -t $(DESTDIR)$(DATADIR)/setarch-aliases.d; done for a in ${SETARCH_ALIASES}; do install -m0644 $$a -t $(DESTDIR)$(DATADIR)/setarch-aliases.d; done
for l in ${COMMITPKG_LINKS}; do ln -sf commitpkg $(DESTDIR)$(PREFIX)/bin/$$l; done for l in ${COMMITPKG_LINKS}; do ln -sf commitpkg $(DESTDIR)$(PREFIX)/bin/$$l; done
@@ -149,8 +142,7 @@ uninstall:
for f in $(notdir $(LIBRARY)); do rm -f $(DESTDIR)$(DATADIR)/lib/$$f; done for f in $(notdir $(LIBRARY)); do rm -f $(DESTDIR)$(DATADIR)/lib/$$f; done
rm -rf $(DESTDIR)$(DATADIR)/lib rm -rf $(DESTDIR)$(DATADIR)/lib
rm -rf $(DESTDIR)$(DATADIR)/git.conf.d rm -rf $(DESTDIR)$(DATADIR)/git.conf.d
rm -rf $(DESTDIR)$(DATADIR)/makepkg.conf.d for conf in $(notdir $(MAKEPKG_CONFIGS)); do rm -f $(DESTDIR)$(DATADIR)/makepkg.conf.d/$${conf##*/}; done
rm -rf $(DESTDIR)$(DATADIR)/data
for conf in $(notdir $(PACMAN_CONFIGS)); do rm -f $(DESTDIR)$(DATADIR)/pacman.conf.d/$${conf##*/}; done for conf in $(notdir $(PACMAN_CONFIGS)); do rm -f $(DESTDIR)$(DATADIR)/pacman.conf.d/$${conf##*/}; done
for f in $(notdir $(SETARCH_ALIASES)); do rm -f $(DESTDIR)$(DATADIR)/setarch-aliases.d/$$f; done for f in $(notdir $(SETARCH_ALIASES)); do rm -f $(DESTDIR)$(DATADIR)/setarch-aliases.d/$$f; done
for l in ${COMMITPKG_LINKS}; do rm -f $(DESTDIR)$(PREFIX)/bin/$$l; done for l in ${COMMITPKG_LINKS}; do rm -f $(DESTDIR)$(PREFIX)/bin/$$l; done
@@ -162,6 +154,7 @@ uninstall:
for manfile in $(notdir $(MANS)); do rm -f $(DESTDIR)$(MANDIR)/man$${manfile##*.}/$${manfile}; done; for manfile in $(notdir $(MANS)); do rm -f $(DESTDIR)$(MANDIR)/man$${manfile##*.}/$${manfile}; done;
rmdir --ignore-fail-on-non-empty \ rmdir --ignore-fail-on-non-empty \
$(DESTDIR)$(DATADIR)/setarch-aliases.d \ $(DESTDIR)$(DATADIR)/setarch-aliases.d \
$(DESTDIR)$(DATADIR)/makepkg.conf.d \
$(DESTDIR)$(DATADIR)/pacman.conf.d \ $(DESTDIR)$(DATADIR)/pacman.conf.d \
$(DESTDIR)$(DATADIR) $(DESTDIR)$(DATADIR)
@@ -194,5 +187,5 @@ coverage: binprogs library conf completion man
check: $(BINPROGS_SRC) $(LIBRARY_SRC) contrib/completion/bash/devtools.in config/makepkg/x86_64.conf contrib/makepkg/PKGBUILD.proto check: $(BINPROGS_SRC) $(LIBRARY_SRC) contrib/completion/bash/devtools.in config/makepkg/x86_64.conf contrib/makepkg/PKGBUILD.proto
shellcheck $^ shellcheck $^
.PHONY: all binprogs library completion conf man data clean install uninstall tag dist upload test coverage check .PHONY: all binprogs library completion conf man clean install uninstall tag dist upload test coverage check
.DELETE_ON_ERROR: .DELETE_ON_ERROR:

View File

@@ -94,7 +94,6 @@ Component: pkgctl db remove
- bat (pretty printing) - bat (pretty printing)
- nvchecker (version checking) - nvchecker (version checking)
- reuse (license compliance)
### Development Dependencies ### Development Dependencies

View File

@@ -2,7 +2,6 @@
/src /src
/*/ /*/
!/keys/ !/keys/
!/LICENSES/
/*.log /*.log
/*.tar.* /*.tar.*

View File

@@ -1,22 +0,0 @@
#!/hint/bash
# shellcheck disable=2034
#
# /etc/makepkg.conf.d/fortran.conf
#
#########################################################################
# FORTRAN LANGUAGE SUPPORT
#########################################################################
# Flags used for the Fortran compiler, similar in spirit to CFLAGS. Read
# linkman:gfortran[1] for more details on the available flags.
FFLAGS="-march=x86-64 -mtune=generic -O2 -pipe -fno-plt \
-Wp,-D_FORTIFY_SOURCE=3 -fstack-clash-protection -fcf-protection \
-fno-omit-frame-pointer -mno-omit-leaf-frame-pointer"
FCFLAGS="$FFLAGS"
# Additional compiler flags appended to `FFLAGS` and `FCFLAGS` for use in debugging. Usually
# this would include: ``-g''. Read linkman:gfortran[1] for more details on the wide
# variety of compiler flags available.
DEBUG_FFLAGS="-g"

View File

@@ -1,19 +0,0 @@
#!/hint/bash
# shellcheck disable=2034
#
# /etc/makepkg.conf.d/rust.conf
#
#########################################################################
# RUST LANGUAGE SUPPORT
#########################################################################
# Flags used for the Rust compiler, similar in spirit to CFLAGS. Read
# linkman:rustc[1] for more details on the available flags.
RUSTFLAGS="-C force-frame-pointers=yes"
# Additional compiler flags appended to `RUSTFLAGS` for use in debugging.
# Usually this would include: ``-C debuginfo=2''. Read linkman:rustc[1] for
# more details on the available flags.
DEBUG_RUSTFLAGS="-C debuginfo=2"

View File

@@ -1 +0,0 @@
../conf.d/fortran.conf

View File

@@ -1 +0,0 @@
../conf.d/rust.conf

View File

@@ -1 +0,0 @@
../conf.d/fortran.conf

View File

@@ -1 +0,0 @@
../conf.d/rust.conf

View File

@@ -127,22 +127,6 @@ _sogrep() { __devtools_complete _sogrep; }
complete -F _sogrep sogrep complete -F _sogrep sogrep
_offload_build_args=(
-r --repo
-a --arch
-s --server
-h --help
)
_offload_build_args__repo_opts() { _devtools_completions_build_repo; }
_offload_build_args_r_opts() { _offload_build_args__repo_opts; }
_offload_build_args__arch_opts() { _devtools_completions_binary_arch; }
_offload_build_args_a_opts() { _offload_build_args__arch_opts; }
_offload_build_args__server_opts() { :; }
_offload_build_args_s_opts() { _offload_build_args__server_opts; }
_offload_build() { __devtools_complete _offload_build; }
complete -F _offload_build offload-build
_pkgctl_cmds=( _pkgctl_cmds=(
aur aur
auth auth
@@ -150,7 +134,6 @@ _pkgctl_cmds=(
db db
diff diff
issue issue
license
release release
repo repo
search search
@@ -368,25 +351,6 @@ _pkgctl_repo_switch_opts() {
fi fi
} }
_pkgctl_license_cmds=(
check
setup
)
_pkgctl_license_check_args=(
-h --help
)
_pkgctl_license_check_opts() { _filedir -d; }
_pkgctl_license_setup_args=(
--no-check
-f --force
-h --help
)
_pkgctl_license_setup_opts() { _filedir -d; }
_pkgctl_version_cmds=( _pkgctl_version_cmds=(
check check
setup setup

View File

@@ -1,4 +1,4 @@
#compdef archbuild arch-nspawn archrelease commitpkg pkgctl diffpkg finddeps makechrootpkg mkarchroot extrapkg=commitpkg corepkg=commitpkg testingpkg=commitpkg stagingpkg=commitpkg communitypkg=commitpkg community-testingpkg=commitpkg community-stagingpkg=commitpkg multilibpkg=commitpkg multilib-testingpkg=commitpkg extra-x86_64-build=archbuild testing-x86_64-build=archbuild staging-x86_64-build=archbuild multilib-build=archbuild multilib-testing-build=archbuild multilib-staging-build=archbuild kde-unstable-x86_64-build=archbuild gnome-unstable-x86_64-build=archbuild checkpkg sogrep offload-build makerepropkg #compdef archbuild arch-nspawn archrelease commitpkg pkgctl diffpkg finddeps makechrootpkg mkarchroot extrapkg=commitpkg corepkg=commitpkg testingpkg=commitpkg stagingpkg=commitpkg communitypkg=commitpkg community-testingpkg=commitpkg community-stagingpkg=commitpkg multilibpkg=commitpkg multilib-testingpkg=commitpkg extra-x86_64-build=archbuild testing-x86_64-build=archbuild staging-x86_64-build=archbuild multilib-build=archbuild multilib-testing-build=archbuild multilib-staging-build=archbuild kde-unstable-x86_64-build=archbuild gnome-unstable-x86_64-build=archbuild checkpkg sogrep makerepropkg
# #
# SPDX-License-Identifier: GPL-3.0-or-later # SPDX-License-Identifier: GPL-3.0-or-later
@@ -61,7 +61,7 @@ _pkgctl_build_args=(
'--update-checksums[Force computation and update of the checksums (disables auto-detection)]' '--update-checksums[Force computation and update of the checksums (disables auto-detection)]'
'(-e --edit)'{-e,--edit}'[Edit the PKGBUILD before building]' '(-e --edit)'{-e,--edit}'[Edit the PKGBUILD before building]'
'(-r --release)'{-r,--release}'[Automatically commit, tag and release after building]' '(-r --release)'{-r,--release}'[Automatically commit, tag and release after building]'
'(-m --message)'{-m,--message}"[Use the given <msg> as the commit message]:message:" '(-m --message=)'{-m,--message=}"[Use the given <msg> as the commit message]:message:"
'(-u --db-update)'{-u,--db-update}'[Automatically update the pacman database as last action]' '(-u --db-update)'{-u,--db-update}'[Automatically update the pacman database as last action]'
'(-h --help)'{-h,--help}'[Display usage]' '(-h --help)'{-h,--help}'[Display usage]'
'*:git_dir:_files -/' '*:git_dir:_files -/'
@@ -201,8 +201,8 @@ _pkgctl_issue_view_args=(
) )
_pkgctl_release_args=( _pkgctl_release_args=(
'(-m --message)'{-m,--message}"[Use the given <msg> as the commit message]:message:" '(-m --message=)'{-m,--message=}"[Use the given <msg> as the commit message]:message:"
'(-r --repo)'{-r,--repo}"[Specify a target repository for new packages]:repo:($DEVTOOLS_VALID_REPOS[*])" '(-r --repo=)'{-r,--repo=}"[Specify a target repository for new packages]:repo:($DEVTOOLS_VALID_REPOS[*])"
'(-s --staging)'{-s,--staging}'[Release to the staging counterpart of the auto-detected repo]' '(-s --staging)'{-s,--staging}'[Release to the staging counterpart of the auto-detected repo]'
'(-t --testing)'{-t,--testing}'[Release to the testing counterpart of the auto-detected repo]' '(-t --testing)'{-t,--testing}'[Release to the testing counterpart of the auto-detected repo]'
'(-u --db-update)'{-u,--db-update}'[Automatically update the pacman database after uploading]' '(-u --db-update)'{-u,--db-update}'[Automatically update the pacman database after uploading]'
@@ -369,13 +369,6 @@ _sogrep_args=(
'2:libname' '2:libname'
) )
_offload_build_args=(
'(-r --repo)'{-r,--repo}'[Build against a specific repository]:repo:($DEVTOOLS_VALID_BUILDREPOS[*])'
'(-a --arch)'{-a,--arch}'[Build against a specific architecture]:arch:(${DEVTOOLS_VALID_BINARY_ARCHES[*]})'
'(-s --server)'{-s,--server}'[Offload to a specific Build server]:server:'
'(-h --help)'{-h,--help}'[Display usage]'
)
_makerepropkg_args=( _makerepropkg_args=(
'-d[Run diffoscope if the package is unreproducible]' '-d[Run diffoscope if the package is unreproducible]'
'-n[Do not run the check() function in the PKGBUILD]' '-n[Do not run the check() function in the PKGBUILD]'
@@ -399,7 +392,6 @@ _pkgctl_cmds=(
"db[Pacman database modification for package update, move etc]" "db[Pacman database modification for package update, move etc]"
"diff[Compare package files using different modes]" "diff[Compare package files using different modes]"
"issue[Work with GitLab packaging issues]" "issue[Work with GitLab packaging issues]"
"license[Check and manage package license compliance]"
"release[Release step to commit, tag and upload build artifacts]" "release[Release step to commit, tag and upload build artifacts]"
"repo[Manage Git packaging repositories and their configuration]" "repo[Manage Git packaging repositories and their configuration]"
"search[Search for an expression across the GitLab packaging group]" "search[Search for an expression across the GitLab packaging group]"
@@ -411,24 +403,6 @@ _pkgctl_args=(
'(-h --help)'{-h,--help}'[Display usage]' '(-h --help)'{-h,--help}'[Display usage]'
) )
_pkgctl_license_cmds=(
"pkgctl license command"
"check[Checks package licensing compliance using REUSE]"
"setup[Automatically detect and setup a basic REUSE config]"
)
_pkgctl_license_check_args=(
'(-h --help)'{-h,--help}'[Display usage]'
'*:git_dir:_files -/'
)
_pkgctl_license_setup_args=(
'(-f --force)'{-f,--force}'[Overwrite existing REUSE config]'
'--no-check[Do not run license check after setup]'
'(-h --help)'{-h,--help}'[Display usage]'
'*:git_dir:_files -/'
)
_pkgctl_version_cmds=( _pkgctl_version_cmds=(
"pkgctl version command" "pkgctl version command"
"check[Compares local package versions against upstream versions]" "check[Compares local package versions against upstream versions]"

View File

@@ -1,12 +0,0 @@
Copyright Arch Linux Contributors
Permission to use, copy, modify, and/or distribute this software for
any purpose with or without fee is hereby granted.
THE SOFTWARE IS PROVIDED “AS IS” AND THE AUTHOR DISCLAIMS ALL
WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES
OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE
FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY
DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

View File

@@ -23,8 +23,7 @@ Options
Location of a pacman config file Location of a pacman config file
*-M* <file>:: *-M* <file>::
Location of a makepkg config file. Specific additions (e.g. build flags for Location of a makepkg config file
additional languages) can be placed in '<file>.d/*.conf'.
*-c* <dir>:: *-c* <dir>::
Set pacman cache, if no directory is specified the passed pacman.conf's cachedir is used with a fallback to '/etc/pacman.conf' Set pacman cache, if no directory is specified the passed pacman.conf's cachedir is used with a fallback to '/etc/pacman.conf'

View File

@@ -58,9 +58,6 @@ makechrootpkg(1)
makerepropkg(1) makerepropkg(1)
Rebuild a package to see if it is reproducible Rebuild a package to see if it is reproducible
offload-build(1)
Build a PKGBUILD on a remote server using makechrootpkg
sogrep(1) sogrep(1)
Find packages using a linked to a given shared library Find packages using a linked to a given shared library

View File

@@ -49,8 +49,7 @@ Options
Set the pacman cache directory. Set the pacman cache directory.
*-M* <file>:: *-M* <file>::
Location of a makepkg config file. Specific additions (e.g. build flags for Location of a makepkg config file.
additional languages) can be placed in '<file>.d/*.conf'.
*-l* <chroot>:: *-l* <chroot>::
The directory name to use as the chroot namespace The directory name to use as the chroot namespace

View File

@@ -1,52 +0,0 @@
offload-build(1)
================
Name
----
offload-build - Build a PKGBUILD on a remote server using makechrootpkg
Synopsis
--------
offload-build [OPTIONS] -- [ARCHBUILD_OPTIONS]
Description
-----------
Build a PKGBUILD on a remote server using makechrootpkg. Requires a remote user
that can run archbuild in a non-interactive manner, e.g. must be able to
elevate permissions using passwordless sudo.
Options
-------
*-r, --repo* <reponame>::
Build against a specific repository. The default is `extra`, to build packages using
the stable repositories via extra-x86_64-build.
*-a, --arch* <architecture>::
Build against a specific architecture. The default is `x86_64`, the only
architecture officially supported by Arch Linux.
*-s, --server* <hostname>::
Offload to a specific build server. The default is build.archlinux.org
which is used as part of the build toolchain for the official Arch Linux
repos.
*-h, --help*::
Show a help text.
Passing options to archbuild
----------------------------
Options after a delimiting -- are passed on to archbuild on the remote.
archbuild in turn supports passing arguments on to makechrootpkg, which in turn
supports passing options to makepkg. Since each uses -- to delimit options that
are forwarded, make sure to escape them properly:
`offload-build offload-args -- archbuild-args -- makechrootpkg-args -- makepkg-args`
Example: To use a second `testing-x86_64-build` instance with another copydir:
`offload-build -r testing -- -- -l <chroot_copy>`
include::include/footer.asciidoc[]

View File

@@ -1,54 +0,0 @@
pkgctl-license-check(1)
=======================
Name
----
pkgctl-license-check - Checks package licensing compliance using REUSE
Synopsis
--------
pkgctl license check [OPTIONS] [PKGBASE...]
Description
-----------
Checks package licensing compliance using REUSE and also verifies whether
a LICENSE file with the expected Arch Linux-specific 0BSD license text exists.
Configuration
-------------
Uses reuse(1) and a `REUSE.toml` file located alongside the PKGBUILD(5). Refer
to the configuration section in pkgctl-license(1).
If no `PKGBASE` is specified, the command defaults to using the current working
directory.
Options
-------
*-h, --help*::
Show a help text
Exit Codes
----------
On exit, return one of the following codes:
*0*::
Normal exit condition, all checked packages are compliant
*1*::
Unknown cause of failure
*2*::
Normal exit condition, but some packages are not compliant
See Also
--------
pkgctl-license(1)
reuse(1)
PKGBUILD(5)
include::include/footer.asciidoc[]

View File

@@ -1,55 +0,0 @@
pkgctl-license-setup(1)
=======================
Name
----
pkgctl-license-setup - Automatically detect and setup a basic REUSE
configuration
Synopsis
--------
pkgctl license setup [OPTIONS] [PKGBASE...]
Description
-----------
This subcommand automates the creation of the Arch Linux 0BSD package license
file as well as a basic reuse(1) configuration by applying simple heuristics.
It comes in especially handy when initially setting up licensing for a package
without the need to manually write a `REUSE.toml` file.
If any `.patch` files are detected and the PKGBUILD(5) has only a single entry
in the `license=()` array, this subcommand assumes the patches are licensed
under that license and generates annotations for them.
In case there are no patches, no additional annotations are generated.
Manual annotations are necessary in case the subcommand can't generate a
configuration that accounts for all files. In this case, `reuse lint` will fail
with a descriptive error of which files are missing an annotation.
If no `PKGBASE` is specified, the command defaults to using the current working
directory.
Options
-------
*-f, --force*::
Overwrite existing reuse(1) configuration
*--no-check*::
Do not run pkgctl-license-check(1) after setup
*-h, --help*::
Show a help text
See Also
--------
pkgctl-license(1)
pkgctl-license-check(1)
reuse(1)
PKGBUILD(5)
include::include/footer.asciidoc[]

View File

@@ -1,54 +0,0 @@
pkgctl-license(1)
=================
Name
----
pkgctl-license - Check and manage package license compliance
Synopsis
--------
pkgctl license [OPTIONS] [SUBCOMMAND]
Description
-----------
Commands related to package licenses, including checks for compliance.
Uses reuse(1) and a `REUSE.toml` file located alongside the PKGBUILD(5).
Configuration
-------------
The `REUSE.toml` file must contain annotations for all regular files expected
to be present in an Arch Linux package repository.
Use pkgctl-license-setup(1) to automatically detect and setup a basic REUSE
config file based on the files in the package repository.
For detailed information on the various configuration options available for the
`REUSE.toml` file, refer to the REUSE Specification (https://reuse.software/spec).
Options
-------
*-h, --help*::
Show a help text
Subcommands
-----------
pkgctl license check::
Checks package licensing compliance using REUSE
pkgctl license setup::
Automatically detect and setup a basic REUSE config
See Also
--------
pkgctl-license-check(1)
pkgctl-license-setup(1)
reuse(1)
PKGBUILD(5)
include::include/footer.asciidoc[]

View File

@@ -112,13 +112,7 @@ copy_hostconf () {
[[ -n $host_cachemirrors ]] && printf 'CacheServer = %s\n' "${host_cachemirrors[@]}" >>"$working_dir/etc/pacman.d/mirrorlist" [[ -n $host_cachemirrors ]] && printf 'CacheServer = %s\n' "${host_cachemirrors[@]}" >>"$working_dir/etc/pacman.d/mirrorlist"
[[ -n $pac_conf ]] && cp "$pac_conf" "$working_dir/etc/pacman.conf" [[ -n $pac_conf ]] && cp "$pac_conf" "$working_dir/etc/pacman.conf"
if [[ -n $makepkg_conf ]]; then [[ -n $makepkg_conf ]] && cp "$makepkg_conf" "$working_dir/etc/makepkg.conf"
cp "$makepkg_conf" "$working_dir/etc/makepkg.conf"
if [[ -d "${makepkg_conf}.d" ]] && is_globfile "${makepkg_conf}.d"/*.conf; then
mkdir --parents "$working_dir/etc/makepkg.conf.d/"
cp "${makepkg_conf}.d/"*.conf "$working_dir/etc/makepkg.conf.d/"
fi
fi
local file local file
for file in "${files[@]}"; do for file in "${files[@]}"; do

View File

@@ -79,13 +79,6 @@ check_root SOURCE_DATE_EPOCH,SRCDEST,SRCPKGDEST,PKGDEST,LOGDEST,MAKEFLAGS,PACKAG
# Pass all arguments after -- right to makepkg # Pass all arguments after -- right to makepkg
makechrootpkg_args+=("${@:$OPTIND}") makechrootpkg_args+=("${@:$OPTIND}")
# Automatically recreate the root chroot if a version mismatch is detected
CURRENT_CHROOT_VERSION=$(cat "${chroots}/${repo}-${arch}/root/.arch-chroot")
if [[ -f "${chroots}/${repo}-${arch}/root/.arch-chroot" ]] && [[ "$CURRENT_CHROOT_VERSION" != "$CHROOT_VERSION" ]]; then
warning "Recreating chroot '%s' (%s) as it is not at version %s" "${chroots}/${repo}-${arch}/root" "$CURRENT_CHROOT_VERSION" "$CHROOT_VERSION"
clean_first=true
fi
if ${clean_first} || [[ ! -d "${chroots}/${repo}-${arch}" ]]; then if ${clean_first} || [[ ! -d "${chroots}/${repo}-${arch}" ]]; then
msg "Creating chroot for [%s] (%s)..." "${repo}" "${arch}" msg "Creating chroot for [%s] (%s)..." "${repo}" "${arch}"

View File

@@ -120,20 +120,8 @@ if (( ${#validpgpkeys[@]} != 0 )); then
git add --force -- keys/pgp/* git add --force -- keys/pgp/*
fi fi
needsversioning=()
if [[ ! -e REUSE.toml || ! -e LICENSE || ! -d LICENSES ]]; then
# TODO: Make this a hard failure in the future after packagers have had
# some time to add licenses to all packages.
warning "package doesn't have proper licensing information, set it up using:"
msg2 'pkgctl license setup'
else
pkgctl license check
needsversioning+=(REUSE.toml LICENSE LICENSES/*)
fi
# find files which should be under source control # find files which should be under source control
needsversioning+=(PKGBUILD) needsversioning=(PKGBUILD)
for s in "${source[@]}"; do for s in "${source[@]}"; do
[[ $s != *://* ]] && needsversioning+=("$s") [[ $s != *://* ]] && needsversioning+=("$s")
done done
@@ -155,7 +143,7 @@ if (( ${#needsversioning[*]} )); then
if [[ ! -f "${file}" ]]; then if [[ ! -f "${file}" ]]; then
continue continue
fi fi
if ! git ls-files --error-unmatch "$file" >/dev/null; then if ! git ls-files --error-unmatch "$file"; then
die "%s is not under version control" "$file" die "%s is not under version control" "$file"
fi fi
done done

View File

@@ -4,7 +4,7 @@
: :
# shellcheck disable=2034 # shellcheck disable=2034
CHROOT_VERSION='v6' CHROOT_VERSION='v5'
## ##
# usage : check_root $keepenv # usage : check_root $keepenv
@@ -15,8 +15,8 @@ check_root() {
local orig_argv=("$@") local orig_argv=("$@")
(( EUID == 0 )) && return (( EUID == 0 )) && return
if type -P run0 && type -P pkexec >/dev/null; then if type -P sudo >/dev/null; then
exec run0 -- "${orig_argv[@]}" exec sudo --preserve-env="${keepenv}" -- "${orig_argv[@]}"
else else
exec su root -c "$(printf ' %q' "${orig_argv[@]}")" exec su root -c "$(printf ' %q' "${orig_argv[@]}")"
fi fi

View File

@@ -8,6 +8,8 @@ DEVTOOLS_INCLUDE_BUILD_SH=1
_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@} _DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@}
# shellcheck source=src/lib/common.sh # shellcheck source=src/lib/common.sh
source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh
# shellcheck source=src/lib/build/offload.sh
source "${_DEVTOOLS_LIBRARY_DIR}"/lib/build/offload.sh
# shellcheck source=src/lib/db/update.sh # shellcheck source=src/lib/db/update.sh
source "${_DEVTOOLS_LIBRARY_DIR}"/lib/db/update.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/db/update.sh
# shellcheck source=src/lib/release.sh # shellcheck source=src/lib/release.sh
@@ -42,10 +44,10 @@ pkgctl_build_usage() {
Build packages inside a clean chroot Build packages inside a clean chroot
Build packages in clean chroot environment, offering various options When a new pkgver is set using the appropriate PKGBUILD options the
and functionalities to customize the package building process. checksums are automatically updated.
By default, chroot environments are located in /var/lib/archbuild/. TODO
BUILD OPTIONS BUILD OPTIONS
--arch ARCH Specify architectures to build for (disables auto-detection) --arch ARCH Specify architectures to build for (disables auto-detection)
@@ -463,7 +465,7 @@ pkgctl_build() {
fi fi
if (( OFFLOAD )); then if (( OFFLOAD )); then
offload-build --repo "${pkgrepo}" -- "${BUILD_OPTIONS[@]}" -- "${MAKECHROOT_OPTIONS[@]}" -l "${WORKER}" -- "${MAKEPKG_OPTIONS[@]}" pkgctl_build_offload_client "${pkgbase}" "${pkgrepo}" "${arch}" "${BUILD_OPTIONS[@]}" -- "${MAKECHROOT_OPTIONS[@]}" -l "${WORKER}" -- "${MAKEPKG_OPTIONS[@]}"
else else
"${BUILDTOOL}" "${BUILD_OPTIONS[@]}" -- "${MAKECHROOT_OPTIONS[@]}" -l "${WORKER}" -- "${MAKEPKG_OPTIONS[@]}" "${BUILDTOOL}" "${BUILD_OPTIONS[@]}" -- "${MAKECHROOT_OPTIONS[@]}" -l "${WORKER}" -- "${MAKEPKG_OPTIONS[@]}"
fi fi
@@ -481,6 +483,9 @@ pkgctl_build() {
# shellcheck disable=SC2119 # shellcheck disable=SC2119
write_srcinfo_file write_srcinfo_file
version=$(get_full_version)
msg "Finished building %s %s" "${pkgbase}" "${version}"
# test-install (some of) the produced packages # test-install (some of) the produced packages
if [[ ${INSTALL_TO_HOST} == auto ]] || [[ ${INSTALL_TO_HOST} == all ]]; then if [[ ${INSTALL_TO_HOST} == auto ]] || [[ ${INSTALL_TO_HOST} == all ]]; then
# shellcheck disable=2119 # shellcheck disable=2119

228
src/lib/build/offload.sh Normal file
View File

@@ -0,0 +1,228 @@
#!/hint/bash
# SPDX-License-Identifier: GPL-3.0-or-later
[[ -z ${DEVTOOLS_INCLUDE_BUILD_OFFLOAD_SH:-} ]] || return 0
DEVTOOLS_INCLUDE_BUILD_OFFLOAD_SH=1
_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@}
# shellcheck source=src/lib/common.sh
source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh
# shellcheck source=src/lib/util/makepkg.sh
source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/makepkg.sh
source /usr/share/makepkg/util/config.sh
source /usr/share/makepkg/util/message.sh
set -eo pipefail
PKGCTL_OFFLOAD_CACHE_HOME="${XDG_CACHE_HOME:-$HOME/.cache}/pkgctl/offload"
pkgctl_build_offload_usage() {
local -r COMMAND=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}}
cat <<- _EOF_
Usage: ${COMMAND} [COMMAND] [OPTIONS]...
Server commands to build packages remotely by offloading the job.
For internal use only!
_EOF_
}
pkgctl_build_offload() {
if (( $# < 1 )); then
pkgctl_build_offload_usage
exit 1
fi
while (( $# )); do
case $1 in
-h|--help)
pkgctl_build_offload_usage
exit 0
;;
create-builddir)
shift
pkgctl_build_offload_server_create_builddir "$@"
exit 0
;;
clean-builddir)
shift
pkgctl_build_offload_server_clean_builddir "$@"
exit 0
;;
build)
shift
pkgctl_build_offload_server_build "$@"
exit 0
;;
collect-files)
shift
pkgctl_build_offload_server_collect_files "$@"
exit 0
;;
collect-logs)
shift
pkgctl_build_offload_server_collect_logs "$@"
exit 0
;;
*)
die "invalid argument: %s" "$1"
;;
esac
done
}
pkgctl_build_offload_client() {
local pkgbase=$1
local pkgrepo=$2
local pkgarch=$3
shift 3
local server=build.archlinux.org
# shellcheck disable=SC2031
local working_dir=$PWD
local _srcpkg srcpkg files
[[ -z ${WORKDIR:-} ]] && setup_workdir
TEMPDIR=$(mktemp --tmpdir="${WORKDIR}" --directory "offload.${pkgbase}.${pkgrepo}-${pkgarch}XXXXXXXXXX")
# Load makepkg.conf variables to be available
# shellcheck disable=SC2119
load_makepkg_config
# Use a source-only tarball as an intermediate to transfer files. This
# guarantees the checksums are okay, and guarantees that all needed files are
# transferred, including local sources, install scripts, and changelogs.
export SRCPKGDEST="${TEMPDIR}"
if ! makepkg_source_package; then
die "unable to make source package"
return 1
fi
# Temporary cosmetic workaround makepkg if SRCDEST is set somewhere else
# but an empty src dir is created in PWD. Remove once fixed in makepkg.
rmdir --ignore-fail-on-non-empty src 2>/dev/null || true
local builddir
builddir=$(
ssh "${SSH_OPTS[@]}" -- "$server" pkgctl offload create-builddir "${pkgbase@Q}" "${pkgrepo@Q}" "${pkgarch@Q}"
)
# Transfer the srcpkg to the server
msg "Transferring source package to the server..."
_srcpkg=("$SRCPKGDEST"/*"$SRCEXT")
srcpkg="${_srcpkg[0]}"
if ! rsync "${RSYNC_OPTS[@]}" -- "$srcpkg" "$server":"${builddir}"; then
die "failed to rsync sources to offload server"
return 1
fi
# Execute build
if ssh "${SSH_OPTS[@]}" -t -- "$server" pkgctl offload build "${builddir@Q}" "${srcpkg@Q}" "${pkgrepo@Q}" "${pkgarch@Q}" "${@@Q}"; then
# Get an array of files that should be downloaded from the server
mapfile -t files < <(
ssh "${SSH_OPTS[@]}" -- "$server" pkgctl offload collect-files "${builddir@Q}" "${pkgrepo@Q}" "${pkgarch@Q}"
)
else
# Build failed, only the logs should be downloaded from the server
mapfile -t files < <(
ssh "${SSH_OPTS[@]}" -- "$server" pkgctl offload collect-logs "${builddir@Q}"
)
fi
# Check if we collected any files to download
if (( ${#files[@]} == 0 )); then
die "failed to collect files to download"
return 1
fi
msg 'Downloading files...'
rsync "${RSYNC_OPTS[@]}" -- "${files[@]/#/$server:}" "${TEMPDIR}/"
# Clean remote build dir
ssh "${SSH_OPTS[@]}" -- "$server" pkgctl offload clean-builddir "${builddir@Q}"
# Move all log files to LOGDEST
if is_globfile "${TEMPDIR}"/*.log; then
mv "${TEMPDIR}"/*.log "${LOGDEST:-${working_dir}}/"
fi
# Assume build failed if we didn't download any package files
if ! is_globfile "${TEMPDIR}"/*.pkg.tar*; then
error "Build failed, check logs in ${LOGDEST:-${working_dir}}"
return 1
fi
# Building a package may change the PKGBUILD during update_pkgver
mv "${TEMPDIR}/PKGBUILD" "${working_dir}/"
mv "${TEMPDIR}"/*.pkg.tar* "${PKGDEST:-${working_dir}}/"
return 0
}
pkgctl_build_offload_server_build() {
local builddir=$1
local srcpkg=$2
local pkgrepo=$3
local pkgarch=$4
shift 4
local buildtool
if [[ -n $pkgarch ]]; then
buildtool="${pkgrepo}-${pkgarch}-build"
else
buildtool="${pkgrepo}-build"
fi
cd "${builddir}"
bsdtar --strip-components 1 -xvf "$(basename "$srcpkg")"
LOGDEST="" "${buildtool}" "$@"
}
pkgctl_build_offload_server_create_builddir() {
local pkgbase=$1
local pkgrepo=$2
local pkgarch=$3
mkdir --parents "${PKGCTL_OFFLOAD_CACHE_HOME}"
mktemp --directory --tmpdir="${PKGCTL_OFFLOAD_CACHE_HOME}" "${pkgbase}.${pkgrepo}-${pkgarch}XXXXXXXXXX"
}
pkgctl_build_offload_server_clean_builddir() {
local builddir=$1
rm --recursive --force -- "${builddir}"
}
pkgctl_build_offload_server_collect_files() {
local builddir=$1
local pkgrepo=$2
local pkgarch=$3
local makepkg_config
local makepkg_user_config
# fallback config for multilib
if [[ ${pkgrepo} == multilib* ]] && [[ -z ${pkgarch} ]]; then
pkgarch=x86_64
fi
cd "${builddir}"
makepkg_user_config="${XDG_CONFIG_HOME:-$HOME/.config}/pacman/makepkg.conf"
makepkg_config="${_DEVTOOLS_LIBRARY_DIR}/makepkg.conf.d/${pkgarch}.conf"
if [[ -f ${_DEVTOOLS_LIBRARY_DIR}/makepkg.conf.d/${pkgrepo}-${pkgarch}.conf ]]; then
makepkg_config="${_DEVTOOLS_LIBRARY_DIR}/makepkg.conf.d/${pkgrepo}-${pkgarch}.conf"
fi
while read -r file; do
if [[ -f "${file}" ]]; then
printf "%s\n" "${file}"
fi
done < <(makepkg --config <(cat "${makepkg_user_config}" "${makepkg_config}" 2>/dev/null) --packagelist)
printf "%s\n" "${builddir}/PKGBUILD"
pkgctl_build_offload_server_collect_logs "${builddir}"
}
pkgctl_build_offload_server_collect_logs() {
local builddir=$1
find "${builddir}" -name "*.log"
}

View File

@@ -18,9 +18,6 @@ export LANG=C.UTF-8
# Avoid systemd trying to color the terminal on systemd-nspawn # Avoid systemd trying to color the terminal on systemd-nspawn
export SYSTEMD_TINT_BACKGROUND=no export SYSTEMD_TINT_BACKGROUND=no
# Avoid diffoscope looking at remote debug info through readelf
unset DEBUGINFOD_URLS
# Set buildtool properties # Set buildtool properties
export BUILDTOOL=devtools export BUILDTOOL=devtools
export BUILDTOOLVER=@buildtoolver@ export BUILDTOOLVER=@buildtoolver@

View File

@@ -1,66 +0,0 @@
#!/hint/bash
#
# SPDX-License-Identifier: GPL-3.0-or-later
[[ -z ${DEVTOOLS_INCLUDE_LICENSE_SH:-} ]] || return 0
DEVTOOLS_INCLUDE_LICENSE_SH=1
_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@}
set -e
pkgctl_license_usage() {
local -r COMMAND=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}}
cat <<- _EOF_
Usage: ${COMMAND} [COMMAND] [OPTIONS]
Check and manage package license compliance.
COMMANDS
check Check package license compliance
setup Automatically detect and setup a basic REUSE config
OPTIONS
-h, --help Show this help text
EXAMPLES
$ ${COMMAND} check libfoo linux libbar
$ ${COMMAND} setup libfoo
_EOF_
}
pkgctl_license() {
if (( $# < 1 )); then
pkgctl_license_usage
exit 0
fi
while (( $# )); do
case $1 in
-h|--help)
pkgctl_license_usage
exit 0
;;
check)
_DEVTOOLS_COMMAND+=" $1"
shift
# shellcheck source=src/lib/license/check.sh
source "${_DEVTOOLS_LIBRARY_DIR}"/lib/license/check.sh
pkgctl_license_check "$@"
exit $?
;;
setup)
_DEVTOOLS_COMMAND+=" $1"
shift
# shellcheck source=src/lib/license/setup.sh
source "${_DEVTOOLS_LIBRARY_DIR}"/lib/license/setup.sh
pkgctl_license_setup "$@"
exit 0
;;
*)
die "invalid argument: %s" "$1"
;;
esac
done
}

View File

@@ -1,147 +0,0 @@
#!/bin/bash
#
# SPDX-License-Identifier: GPL-3.0-or-later
#
[[ -z ${DEVTOOLS_INCLUDE_LICENSE_CHECK_SH:-} ]] || return 0
DEVTOOLS_INCLUDE_LICENSE_CHECK_SH=1
_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@}
# shellcheck source=src/lib/common.sh
source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh
source /usr/share/makepkg/util/message.sh
set -eo pipefail
readonly PKGCTL_LICENSE_CHECK_EXIT_COMPLIANT=0
export PKGCTL_LICENSE_CHECK_EXIT_COMPLIANT
readonly PKGCTL_LICENSE_CHECK_EXIT_FAILURE=2
export PKGCTL_LICENSE_CHECK_EXIT_FAILURE
pkgctl_license_check_usage() {
local -r COMMAND=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}}
cat <<- _EOF_
Usage: ${COMMAND} [OPTIONS] [PKGBASE]...
Checks package licensing compliance using REUSE and also verifies
whether a LICENSE file with the expected Arch Linux-specific 0BSD
license text exists.
Upon execution, it runs 'reuse lint'.
OPTIONS
-h, --help Show this help text
EXAMPLES
$ ${COMMAND} neovim vim
_EOF_
}
pkgctl_license_check() {
local pkgbases=()
local verbose=0
local license_text
license_text=$(< "${_DEVTOOLS_LIBRARY_DIR}"/data/LICENSE)
local exit_code=${PKGCTL_LICENSE_CHECK_EXIT_COMPLIANT}
while (( $# )); do
case $1 in
-h|--help)
pkgctl_license_check_usage
exit 0
;;
--)
shift
break
;;
-*)
die "invalid argument: %s" "$1"
;;
*)
pkgbases=("$@")
break
;;
esac
done
if ! command -v reuse &>/dev/null; then
die "The \"$_DEVTOOLS_COMMAND\" command requires the 'reuse' CLI tool"
fi
# Check if used without pkgbases in a packaging directory
if (( ${#pkgbases[@]} == 0 )); then
if [[ -f PKGBUILD ]]; then
pkgbases=(".")
else
pkgctl_license_check_usage
exit 1
fi
fi
# enable verbose mode when we only have a single item to check
if (( ${#pkgbases[@]} == 1 )); then
verbose=1
fi
for path in "${pkgbases[@]}"; do
# skip paths that are not directories
if [[ ! -d "${path}" ]]; then
continue
fi
pushd "${path}" >/dev/null
if [[ ! -f PKGBUILD ]]; then
msg_error "${BOLD}${pkgbase}:${ALL_OFF} no PKGBUILD found"
return 1
fi
# reset common PKGBUILD variables
unset pkgbase
# shellcheck source=contrib/makepkg/PKGBUILD.proto
if ! . ./PKGBUILD; then
msg_error "${BOLD}${pkgbase}:${ALL_OFF} failed to source PKGBUILD"
return 1
fi
pkgbase=${pkgbase:-$pkgname}
if [[ ! -e LICENSE ]]; then
msg_error "${BOLD}${pkgbase}:${ALL_OFF} is missing the LICENSE file"
return "${PKGCTL_LICENSE_CHECK_EXIT_FAILURE}"
fi
if [[ ! -L LICENSES/0BSD.txt ]]; then
msg_error "${BOLD}${pkgbase}:${ALL_OFF} LICENSES/0BSD should be a symlink to LICENSE but it isn't"
return "${PKGCTL_LICENSE_CHECK_EXIT_FAILURE}"
fi
# Check if the local LICENSE file mismatches our expectations
if [[ $license_text != $(< LICENSE) ]]; then
msg_error "${BOLD}${pkgbase}:${ALL_OFF} LICENSE file doesn't have the expected Arch Linux-specific license text"
return "${PKGCTL_LICENSE_CHECK_EXIT_FAILURE}"
fi
# Check for REUSE compliance
if ! reuse lint --json | jq --exit-status '.summary.compliant' &>/dev/null; then
msg_error "${BOLD}${pkgbase}:${ALL_OFF} repository is not REUSE compliant"
exit_code=${PKGCTL_LICENSE_CHECK_EXIT_FAILURE}
# re-execute reuse lint for human readable output
if (( verbose )); then
reuse lint
fi
popd >/dev/null
continue
fi
msg_success "${BOLD}${pkgbase}:${ALL_OFF} repository is REUSE compliant"
popd >/dev/null
done
# return status based on results
return "${exit_code}"
}

View File

@@ -1,272 +0,0 @@
#!/bin/bash
#
# SPDX-License-Identifier: GPL-3.0-or-later
#
[[ -z ${DEVTOOLS_INCLUDE_LICENSE_SETUP_SH:-} ]] || return 0
DEVTOOLS_INCLUDE_LICENSE_SETUP_SH=1
_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@}
# shellcheck source=src/lib/common.sh
source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh
# shellcheck source=src/lib/license/check.sh
source "${_DEVTOOLS_LIBRARY_DIR}"/lib/license/check.sh
source /usr/share/makepkg/util/message.sh
set -eo pipefail
shopt -s nullglob
pkgctl_license_setup_usage() {
local -r COMMAND=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}}
cat <<- _EOF_
Usage: ${COMMAND} [OPTIONS] [PKGBASE]...
Automate the creation of a basic REUSE configuration by analyzing the
license array specified in the PKGBUILD file of a package.
If no PKGBASE is specified, the command defaults to using the current
working directory.
OPTIONS
-f, --force Overwrite existing REUSE configuration
-h, --help Show this help text
--no-check Do not run license check after setup
EXAMPLES
$ ${COMMAND} neovim vim
_EOF_
}
pkgctl_license_setup() {
local pkgbases=()
local force=0
local run_check=1
local path exit_code
local checks=()
while (( $# )); do
case $1 in
-h|--help)
pkgctl_license_setup_usage
exit 0
;;
-f|--force)
force=1
shift
;;
--no-check)
run_check=0
shift
;;
--)
shift
break
;;
-*)
die "invalid argument: %s" "$1"
;;
*)
pkgbases=("$@")
break
;;
esac
done
if ! command -v reuse &>/dev/null; then
die "The \"$_DEVTOOLS_COMMAND\" command requires the 'reuse' CLI tool"
fi
# Check if used without pkgbases in a packaging directory
if (( ${#pkgbases[@]} == 0 )); then
if [[ -f PKGBUILD ]]; then
pkgbases=(".")
else
pkgctl_license_setup_usage
exit 1
fi
fi
exit_code=0
for path in "${pkgbases[@]}"; do
# skip paths that are not directories
if [[ ! -d "${path}" ]]; then
continue
fi
pushd "${path}" >/dev/null
if license_setup "${path}" "${force}"; then
checks+=("${path}")
else
exit_code=1
fi
popd >/dev/null
done
# run checks on the setup targets
if (( run_check )) && (( ${#checks[@]} >= 1 )); then
echo
echo 📡 Running checks...
pkgctl_license_check "${checks[@]}" || true
fi
return "$exit_code"
}
license_setup() {
local path=$1
local force=$2
local pkgbase pkgname license
if [[ ! -f PKGBUILD ]]; then
msg_error "${BOLD}${path}:${ALL_OFF} no PKGBUILD found"
return 1
fi
# shellcheck source=contrib/makepkg/PKGBUILD.proto
if ! . ./PKGBUILD; then
msg_error "${BOLD}${path}:${ALL_OFF} failed to source PKGBUILD"
return 1
fi
pkgbase=${pkgbase:-$pkgname}
# setup LICENSE file
if ! license_file_setup "${pkgbase}" "${force}"; then
return 1
fi
# setup REUSE.toml
if ! reuse_setup "${pkgbase}" "${force}" "${license[@]}"; then
return 1
fi
}
is_valid_license() {
local license_name=$1
local supported_licenses
supported_licenses=$(reuse supported-licenses | awk '{print $1}')
if grep --quiet "^${license_name}$" <<< "$supported_licenses"; then
return 0
else
return 1
fi
}
license_file_setup() {
local pkgbase=$1
local force=$2
local license_text
license_text=$(< "${_DEVTOOLS_LIBRARY_DIR}"/data/LICENSE)
# Write LICENSE file, or check if it mismatches
if (( force )) || [[ ! -f LICENSE ]]; then
printf "%s\n" "${license_text}" > LICENSE
msg_success "${BOLD}${pkgbase}:${ALL_OFF} successfully configured LICENSE"
elif [[ -f LICENSE ]]; then
# if there is a license file, check whether it has the text we expect
existing_license="$(< LICENSE)"
if [[ "${license_text}" != "${existing_license}" ]]; then
msg_error "${BOLD}${pkgbase}:${ALL_OFF} existing LICENSE file doesn't have expected content, use --force to overwrite"
return 1
fi
fi
# make sure the LICENSE file is found by REUSE
mkdir --parents LICENSES
ln --symbolic --force ../LICENSE LICENSES/0BSD.txt
}
reuse_default_annotations() {
cat << EOF
version = 1
[[annotations]]
path = [
"PKGBUILD",
"README.md",
"keys/**",
".SRCINFO",
".gitignore",
".nvchecker.toml",
"*.install",
"*.sysusers",
"*sysusers.conf",
"*.tmpfiles",
"*tmpfiles.conf",
"*.logrotate",
"*.pam",
"*.service",
"*.socket",
"*.timer",
"*.desktop",
"*.hook",
]
SPDX-FileCopyrightText = "Arch Linux contributors"
SPDX-License-Identifier = "0BSD"
EOF
}
reuse_setup() {
local pkgbase=$1
local force=$2
shift 2
local license=("$@")
# Check if REUSE.toml already exists
if (( ! force )) && [[ -f REUSE.toml ]]; then
msg_error "${BOLD}${pkgbase}:${ALL_OFF} REUSE.toml already exists, use --force to overwrite"
return 1
fi
reuse_default_annotations > REUSE.toml
local warning_occurred=0
local patches=(*.patch)
# If there are patches and there's only a single well-known license listed in the package,
# we can generate the annotations for the patches, otherwise we will fail to do so and warn
# the user.
if (( ${#patches} )); then
# If there are multiple licenses, we can't make a good guess about which license the
# patches should have. In case the first element contains a space, we are dealing with
# a complex SPDX license identifier.
if (( ${#license[@]} > 1 )) || [[ ${license[0]} =~ [[:space:]] ]]; then
msg_warn "${BOLD}${pkgbase}:${ALL_OFF} .patch files were found but couldn't automatically guess a suitable license because PKGBUILD has multiple licenses"
patch_annotations "${pkgbase}" "TODO-Choose-a-license" "${patches[@]}" >> REUSE.toml
warning_occurred=1
elif ! is_valid_license "${license[0]}"; then
msg_warn "${BOLD}${pkgbase}:${ALL_OFF} .patch files were found but couldn't automatically guess a suitable license because the PKGBUILD license '${license[0]}' is not a recognized SPDX license"
patch_annotations "${pkgbase}" "TODO-Choose-a-license" "${patches[@]}" >> REUSE.toml
warning_occurred=1
else
patch_annotations "${pkgbase}" "${license[0]}" "${patches[@]}" >> REUSE.toml
fi
fi
if (( warning_occurred )); then
msg_warn "${BOLD}${pkgbase}:${ALL_OFF} configured REUSE but a warning occurred, manually edit and fix REUSE.toml"
return 1
else
reuse download --all
msg_success "${BOLD}${pkgbase}:${ALL_OFF} successfully configured REUSE.toml"
fi
}
patch_annotations() {
local pkgbase=$1
local license_identifier=$2
shift 2
local patches=("$@")
local annotations
annotations+="\n[[annotations]]\n"
annotations+="path = [\n"
for patch in "${patches[@]}"; do
annotations+=" \"$(basename "${patch}")\",\n"
done
annotations+="]\n"
annotations+="SPDX-FileCopyrightText = \"${pkgbase} contributors\"\n"
annotations+="SPDX-License-Identifier = \"${license_identifier}\""
echo -e "${annotations}"
}

View File

@@ -93,36 +93,14 @@ get_makepkg_conf() {
local fname=${1} local fname=${1}
local arch="${2}" local arch="${2}"
local makepkg_conf="${3}" local makepkg_conf="${3}"
if ! buildtool_file=$(get_pkgfile "${fname}"); then if ! buildtool_file=$(get_pkgfile "${fname}"); then
error "failed to retrieve ${fname}" error "failed to retrieve ${fname}"
return 1
fi
buildtool_file="${buildtool_file/file:\/\//}"
msg "using makepkg.conf from ${fname}"
# try to handle config of legacy devtools
if bsdtar --list --file "${buildtool_file}" "usr/share/devtools/makepkg-${arch}.conf" &>/dev/null; then
bsdtar --extract --to-stdout --fast-read --file "${buildtool_file}" "usr/share/devtools/makepkg-${arch}.conf" > "${makepkg_conf}"
return $?
fi
msg2 "extracting ${arch}.conf from devtools archive"
if ! bsdtar --extract --to-stdout --fast-read --file "${buildtool_file}" "usr/share/devtools/makepkg.conf.d/${arch}.conf" > "${makepkg_conf}"; then
error "failed to extract 'usr/share/devtools/makepkg.conf.d/${arch}.conf' from devtools archive"
return 1 return 1
fi fi
msg2 "using makepkg.conf from ${fname}"
mkdir --parents "${makepkg_conf}.d" if ! bsdtar xOqf "${buildtool_file/file:\/\//}" "usr/share/devtools/makepkg.conf.d/${arch}.conf" > "${makepkg_conf}"; then
if bsdtar --list --file "${buildtool_file}" "usr/share/devtools/makepkg.conf.d/conf.d" &>/dev/null; then bsdtar xOqf "${buildtool_file/file:\/\//}" "usr/share/devtools/makepkg-${arch}.conf" > "${makepkg_conf}"
msg2 "extracting conf.d from devtools archive" fi
bsdtar --extract --file "${buildtool_file}" --cd "${makepkg_conf}.d" --strip-components 4 "usr/share/devtools/makepkg.conf.d/conf.d"
fi
if bsdtar --list --file "${buildtool_file}" "usr/share/devtools/makepkg.conf.d/${arch}.conf.d" &>/dev/null; then
msg2 "extracting ${arch}.conf.d from devtools archive"
bsdtar --extract --file "${buildtool_file}" --cd "${makepkg_conf}.d" --strip-components 4 "usr/share/devtools/makepkg.conf.d/${arch}.conf.d"
fi
return 0 return 0
} }
@@ -208,7 +186,7 @@ for f in "${splitpkgs[@]}"; do
done done
if (( ${#cache_dirs[@]} == 0 )); then if (( ${#cache_dirs[@]} == 0 )); then
mapfile -t cache_dirs < <(pacman-conf CacheDir) mapfile -t cache_dirs < <(pacman-conf CacheDir)
fi fi
ORIG_HOME=${HOME} ORIG_HOME=${HOME}

View File

@@ -1,162 +0,0 @@
#!/bin/bash
#
# offload-build - build a PKGBUILD on a remote server using makechrootpkg.
#
# Copyright (c) 2019 by Eli Schwartz <eschwartz@archlinux.org>
#
# SPDX-License-Identifier: GPL-3.0-or-later
_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@}
# shellcheck source=src/lib/common.sh
source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh
# shellcheck source=src/lib/util/makepkg.sh
source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/makepkg.sh
source /usr/share/makepkg/util/config.sh
# Deprecation warning
if [[ -z $_DEVTOOLS_COMMAND ]]; then
warning "${0##*/} is deprecated and will be removed. Use 'pkgctl build --offload' instead"
fi
# global defaults suitable for use by Arch staff
repo=extra
arch=x86_64
server=build.archlinux.org
usage() {
cat <<- _EOF_
Usage: ${BASH_SOURCE[0]##*/} [--repo REPO] [--arch ARCHITECTURE] [--server SERVER] -- [ARCHBUILD_ARGS]
Build a PKGBUILD on a remote server using makechrootpkg. Requires a remote user
that can run archbuild without password auth. Options passed after a -- are
passed on to archbuild, and eventually to makechrootpkg.
OPTIONS
-r, --repo Build against a specific repository (current: $repo)
-a, --arch Build against a specific architecture (current: $arch)
-s, --server Offload to a specific build server (current: $server)
-h, --help Show this help text
_EOF_
}
# option checking
while (( $# )); do
case $1 in
-h|--help)
usage
exit 0
;;
-r|--repo)
repo=$2
shift 2
;;
-a|--arch)
arch=$2
shift 2
;;
-s|--server)
server=$2
shift 2
;;
--)
shift
break
;;
*)
die "invalid argument: %s" "$1"
;;
esac
done
# multilib must be handled specially
archbuild_arch="${arch}"
if [[ $repo = multilib* ]]; then
archbuild_arch=
fi
archbuild_cmd=("${repo}${archbuild_arch:+-$archbuild_arch}-build" "$@")
[[ -z ${WORKDIR:-} ]] && setup_workdir
export TEMPDIR=$(mktemp --tmpdir="${WORKDIR}" --directory offload-build.XXXXXXXXXX)
# Load makepkg.conf variables to be available
load_makepkg_config
# Use a source-only tarball as an intermediate to transfer files. This
# guarantees the checksums are okay, and guarantees that all needed files are
# transferred, including local sources, install scripts, and changelogs.
export SRCPKGDEST="${TEMPDIR}"
makepkg_source_package || die "unable to make source package"
# Temporary cosmetic workaround makepkg if SRCDEST is set somewhere else
# but an empty src dir is created in PWD. Remove once fixed in makepkg.
rmdir --ignore-fail-on-non-empty src 2>/dev/null || true
# Create a temporary directory on the server
remote_temp=$(
ssh "${SSH_OPTS[@]}" -- "$server" '
temp="${XDG_CACHE_HOME:-$HOME/.cache}/offload-build" &&
mkdir -p "$temp" &&
mktemp --directory --tmpdir="$temp"
')
# Transfer the srcpkg to the server
msg "Transferring source package to the server..."
_srcpkg=("$SRCPKGDEST"/*"$SRCEXT")
srcpkg="${_srcpkg[0]}"
rsync "${RSYNC_OPTS[@]}" -- "$srcpkg" "$server":"$remote_temp" || die
# Prepare the srcpkg on the server
msg "Extracting srcpkg"
ssh "${SSH_OPTS[@]}" -- "$server" "cd ${remote_temp@Q} && bsdtar --strip-components 1 -xvf $(basename "$srcpkg")" || die
# Run the build command on the server
msg "Running archbuild"
# shellcheck disable=SC2145
if ssh "${SSH_OPTS[@]}" -t -- "$server" "cd ${remote_temp@Q} && export LOGDEST="" && ${archbuild_cmd[@]@Q}"; then
msg "Build complete"
# Get an array of files that should be downloaded from the server
mapfile -t files < <(
ssh "${SSH_OPTS[@]}" -- "$server" "
cd ${remote_temp@Q}"' &&
makepkg_user_config="${XDG_CONFIG_HOME:-$HOME/.config}/pacman/makepkg.conf" &&
makepkg_config="/usr/share/devtools/makepkg.conf.d/'"${arch}"'.conf" &&
if [[ -f /usr/share/devtools/makepkg.conf.d/'"${repo}"'-'"${arch}"'.conf ]]; then
makepkg_config="/usr/share/devtools/makepkg.conf.d/'"${repo}"'-'"${arch}"'.conf"
fi &&
while read -r file; do
[[ -f "${file}" ]] && printf "%s\n" "${file}" ||:
done < <(makepkg --config <(cat "${makepkg_user_config}" "${makepkg_config}" 2>/dev/null) --packagelist) &&
printf "%s\n" '"${remote_temp@Q}/PKGBUILD"'
find '"${remote_temp@Q}"' -name "*.log"
')
else
# Build failed, only the logs should be downloaded from the server
mapfile -t files < <(
ssh "${SSH_OPTS[@]}" -- "$server" '
find '"${remote_temp@Q}"' -name "*.log"
')
fi
if (( ${#files[@]} )); then
msg 'Downloading files...'
rsync "${RSYNC_OPTS[@]}" -- "${files[@]/#/$server:}" "${TEMPDIR}/" || die
if is_globfile "${TEMPDIR}"/*.log; then
mv "${TEMPDIR}"/*.log "${LOGDEST:-${PWD}}/"
fi
if is_globfile "${TEMPDIR}"/*.pkg.tar*; then
# Building a package may change the PKGBUILD during update_pkgver
mv "${TEMPDIR}/PKGBUILD" "${PWD}/"
mv "${TEMPDIR}"/*.pkg.tar* "${PKGDEST:-${PWD}}/"
else
error "Build failed, check logs in ${LOGDEST:-${PWD}}"
exit 1
fi
else
exit 1
fi

View File

@@ -25,7 +25,6 @@ usage() {
db Pacman database modification for package update, move etc db Pacman database modification for package update, move etc
diff Compare package files using different modes diff Compare package files using different modes
issue Work with GitLab packaging issues issue Work with GitLab packaging issues
license Check and manage package licenses
release Release step to commit, tag and upload build artifacts release Release step to commit, tag and upload build artifacts
repo Manage Git packaging repositories and their configuration repo Manage Git packaging repositories and their configuration
search Search for an expression across the GitLab packaging group search Search for an expression across the GitLab packaging group
@@ -114,12 +113,12 @@ while (( $# )); do
pkgctl_issue "$@" pkgctl_issue "$@"
exit 0 exit 0
;; ;;
license) offload)
_DEVTOOLS_COMMAND+=" $1" _DEVTOOLS_COMMAND+=" $1"
shift shift
# shellcheck source=src/lib/license.sh # shellcheck source=src/lib/build/offload.sh
source "${_DEVTOOLS_LIBRARY_DIR}"/lib/license.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/build/offload.sh
pkgctl_license "$@" pkgctl_build_offload "$@"
exit 0 exit 0
;; ;;
release) release)