newrepo (#29)

buildtree: only -n creates repos

buildtree: move msg in sync func

buildtree: deprecate pkglists

Makefile

@@ -1,4 +1,4 @@
-VERSION=0.17
+VERSION=0.18
 
 CHROOT_VERSION=0.10
 
@@ -14,7 +14,6 @@ BASE_CONF = \
 	data/conf/artools-base.conf
 
 BASE_BIN = \
-	bin/base/signfile \
 	bin/base/chroot-run \
 	bin/base/mkchroot \
 	bin/base/basestrap \
@@ -45,7 +44,8 @@ PKG_BIN = \
 	bin/pkg/checkpkg \
 	bin/pkg/finddeps \
 	bin/pkg/find-libdeps \
-	bin/pkg/batchpkg
+	bin/pkg/batchpkg \
+	bin/pkg/signpkg
 
 LN_COMMITPKG = \
 	extrapkg \

bin/base/chroot-run.in

@@ -147,7 +147,7 @@ elif [[ $(cat "$working_dir/.artools") != ${CHROOTVERSION} ]]; then
     die "chroot '%s' is not at version %s. Please rebuild." "$working_dir" "${CHROOTVERSION}"
 fi
 
-sync_host_localtime
+#sync_host_localtime
 
 chroot_api_mount "${working_dir}" || die "failed to setup API filesystems in chroot %s" "${working_dir}"
 

bin/base/signfile.in

@@ -1,32 +0,0 @@
-#!/bin/bash
-#
-# Copyright (C) 2018-19 artoo@artixlinux.org
-# Copyright (C) 2018 Artix Linux Developers
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-
-. @libdir@/artools/util-base.sh
-
-load_makepkg_config
-
-file_to_sign="$1"
-
-if [ ! -e "$1" ]; then
-    error "%s does not exist!" "$file_to_sign"
-    exit 1
-fi
-
-if [[ -n "${BUILDBOT_GPGP}" ]]; then
-    msg2 "Signing [%s]" "${file_to_sign##*/}"
-    gpg --batch --passphrase "${BUILDBOT_GPGP}" --detach-sign "$file_to_sign"
-else
-    msg2 "Signing [%s] with key %s" "${file_to_sign##*/}" "${GPGKEY}..."
-    gpg --detach-sign --use-agent -u "${GPGKEY}" "$file_to_sign"
-fi

bin/pkg/buildpkg.in

@@ -30,6 +30,7 @@ base_devel=('base-devel')
 usage() {
     echo "Usage: ${0##*/} [options] -- [mkchrootpkg_args]"
     echo "    -r <dir>           Create chroots in this directory"
+    echo "    -d <dir>           Destination repo chroot"
     echo '    -c                 Recreate the chroot before building'
     echo '    -m                 Major rebuild'
     echo '    -N                 Disable check() function'
@@ -43,11 +44,12 @@ usage() {
 
 orig_argv=("$0" "$@")
 
-opts='hcCNmr:'
+opts='hcCNmr:d:'
 
 while getopts "${opts}" arg; do
     case "${arg}" in
         r) CHROOTS_PKG="$OPTARG" ;;
+        d) repo="$OPTARG" ;;
         c) create_first=true ;;
         m) rebuild=true ;;
         C) is_checkpkg=true; mkchrootpkg_args+=(-C) ;;

bin/pkg/buildtree.in

@@ -85,7 +85,7 @@ pull_tree_artix(){
 sync_pkg(){
     local rsync_args=(-aWxvci --progress --delete-before --no-R --no-implied-dirs)
     local src="$1" dest="$2"
-
+    msg "Sync from Arch [%s] to Artix [%s]" "${src##*archlinux/}" "${dest##*artixlinux/}"
     rsync "${rsync_args[@]}" $src/ $dest/
 }
 
@@ -111,84 +111,24 @@ show_deps(){
 }
 
 subrepo_new(){
-    local group="${1:-$GROUP}" team="${2:-$TEAM}"
-    local dest=${TREE_DIR_ARTIX}/$group/${PACKAGE}/trunk
+    cd ${TREE_DIR_ARTIX}/${GROUP}
 
-    cd ${TREE_DIR_ARTIX}/$group
-
-    msg2 "team: %s" "$team"
-    msg2 "group: %s" "$group"
+    msg2 "team: %s" "${TEAM}"
+    msg2 "group: %s" "${GROUP}"
 
     local org=$(get_pkg_org "${PACKAGE}")
 
     create_repo "${PACKAGE}" "$org"
 
-    add_repo_to_team "${PACKAGE}" "$org" "$team"
+    add_repo_to_team "${PACKAGE}" "$org" "${TEAM}"
 
     subrepo_clone "${PACKAGE}" "$org"
 
-    prepare_dir "$dest"
+    prepare_dir "${TREE_DIR_ARTIX}/${GROUP}/${PACKAGE}/trunk"
 
     commit_jenkins_files "${PACKAGE}"
 }
 
-get_group_packages(){
-    local pkglist="${SYSCONFDIR}/pkglist.d/$1.list"
-
-    local _space="s| ||g" _clean=':a;N;$!ba;s/\n/ /g' _com_rm="s|#.*||g"
-
-    local pkgs=($(sed "$_com_rm" "$pkglist" | sed "$_space" | sed "$_clean"))
-
-    local cases=
-    for p in ${pkgs[@]};do
-        cases=${cases:-}${cases:+|}${p}
-    done
-    echo $cases
-}
-
-get_group(){
-    local pkg="$1" fallback="${2##*/}" tree=
-    eval "case $pkg in
-        $(get_group_packages kernel)) tree=packages-kernel ;;
-        $(get_group_packages net)) tree=packages-net ;;
-        $(get_group_packages python)) tree=packages-python ;;
-        $(get_group_packages perl)) tree=packages-perl ;;
-        $(get_group_packages ruby)) tree=packages-ruby ;;
-        $(get_group_packages openrc)) tree=packages-openrc ;;
-        $(get_group_packages runit)) tree=packages-runit ;;
-        $(get_group_packages s6)) tree=packages-s6 ;;
-        $(get_group_packages media)) tree=packages-media ;;
-        $(get_group_packages xorg)) tree=packages-xorg ;;
-        $(get_group_packages qt5)) tree=packages-qt5 ;;
-        $(get_group_packages gtk)) tree=packages-gtk ;;
-        $(get_group_packages java)) tree=packages-java ;;
-        $(get_group_packages haskell)) tree=packages-haskell ;;
-        $(get_group_packages devel)) tree=packages-devel ;;
-        $(get_group_packages lxqt)) tree=packages-lxqt ;;
-        $(get_group_packages cinnamon)) tree=packages-cinnamon ;;
-        $(get_group_packages kde)) tree=packages-kde ;;
-        $(get_group_packages gnome)) tree=packages-gnome ;;
-        $(get_group_packages mate)) tree=packages-mate ;;
-        $(get_group_packages xfce)) tree=packages-xfce ;;
-        $(get_group_packages wm)) tree=packages-wm ;;
-        *) tree=$fallback ;;
-    esac"
-    echo $tree
-}
-
-get_team(){
-    local repo="$1" team=
-    case $repo in
-        *testing*|*staging*)
-            set_arch_repos false false false
-            team=$(find_repo "$srcpath")
-            team=${team%-*}
-        ;;
-        *) team=${repo%-*} ;;
-    esac
-    echo $team
-}
-
 from_arch(){
     cd ${TREE_DIR_ARTIX}
 
@@ -196,33 +136,29 @@ from_arch(){
     [[ -z $srcpath ]] && die "Package '%s' does not exist!" "${PACKAGE}"
 
     local repo=$(find_repo "$srcpath")
-
-    archpath=$srcpath/repos/$repo
-    local artixpath=$(find_pkg ${TREE_DIR_ARTIX} ${PACKAGE})
-    local tree=${srcpath%/*}
-    local tree=${tree##*/}
-    local group=$(get_group "${PACKAGE}" "$tree")
-
     msg "repo: %s" "$repo"
-    artixpath=$artixpath/trunk
+
+    local archpath=$srcpath/repos/$repo
+    local artixpath=$(find_pkg ${TREE_DIR_ARTIX} ${PACKAGE})
 
     show_deps "$archpath"
 
+    if [[ -f "$artixpath"/.gitrepo ]]; then
+        artixpath=$artixpath/trunk
+
         if [[ -f $artixpath/PKGBUILD ]];then
 
             . $artixpath/PKGBUILD 2>/dev/null
             local artixver=$(get_full_version)
 
             msg2 "Artix Version: %s" "$artixver"
-    else
-        artixpath=${TREE_DIR_ARTIX}/$group/${PACKAGE}/trunk
-        subrepo_new "${group}" "$(get_team $repo)"
-    fi
-
-    msg "Sync (%s) from Arch [%s] to Artix [%s]" "${PACKAGE}" "${archpath##*archlinux/}" "${artixpath##*artixlinux/}"
 
             sync_pkg "$archpath" "$artixpath"
             patch_pkg "$artixpath"
+        fi
+    else
+        die "Package '%s' does not exist!" "${PACKAGE}"
+    fi
 }
 
 view_build(){
@@ -288,7 +224,7 @@ usage() {
     echo "    -a            Don't clone or pull arch repos"
     echo "    -b            Don't clone or pull artix repos"
     echo '    -i            Import a package from arch repos'
-    echo '    -n            Make new remote subrepo and clone it'
+    echo '    -n            New remote subrepo and clone it'
     echo '    -v            Check if a package is in the artix tree(s)'
     echo '    -c            View package depends'
     echo '    -x            Exclude testing (only with -i)'

bin/pkg/checkpkg.in

@@ -18,6 +18,68 @@ shopt -s extglob
 
 load_makepkg_config
 
+pkgver_equal() {
+    if [[ $1 = *-* && $2 = *-* ]]; then
+        # if both versions have a pkgrel, then they must be an exact match
+        [[ $1 = "$2" ]]
+    else
+        # otherwise, trim any pkgrel and compare the bare version.
+        [[ ${1%%-*} = "${2%%-*}" ]]
+    fi
+}
+
+find_cached_package() {
+    local searchdirs=("$PKGDEST" "$PWD") results=()
+    local targetname=$1 targetver=$2 targetarch=$3
+    local dir pkg pkgbasename name ver rel arch r results
+
+    for dir in "${searchdirs[@]}"; do
+        [[ -d $dir ]] || continue
+
+        for pkg in "$dir"/*.pkg.tar?(.!(sig|*.*)); do
+            [[ -f $pkg ]] || continue
+
+            # avoid adding duplicates of the same inode
+            for r in "${results[@]}"; do
+                [[ $r -ef $pkg ]] && continue 2
+            done
+
+            # split apart package filename into parts
+            pkgbasename=${pkg##*/}
+            pkgbasename=${pkgbasename%.pkg.tar*}
+
+            arch=${pkgbasename##*-}
+            pkgbasename=${pkgbasename%-"$arch"}
+
+            rel=${pkgbasename##*-}
+            pkgbasename=${pkgbasename%-"$rel"}
+
+            ver=${pkgbasename##*-}
+            name=${pkgbasename%-"$ver"}
+
+            if [[ $targetname = "$name" && $targetarch = "$arch" ]] &&
+                pkgver_equal "$targetver" "$ver-$rel"; then
+                results+=("$pkg")
+            fi
+        done
+    done
+
+    case ${#results[*]} in
+        0)
+            return 1
+        ;;
+        1)
+            printf '%s\n' "${results[0]}"
+            return 0
+        ;;
+        *)
+            error 'Multiple packages found:'
+            printf '\t%s\n' "${results[@]}" >&2
+            return 1
+        ;;
+    esac
+}
+
 usage() {
     cat <<- _EOF_
         Usage: ${BASH_SOURCE[0]##*/} [OPTIONS]

bin/pkg/deploypkg.in

@@ -16,35 +16,11 @@
 
 prepare_artools
 
-find_cached_pkgfile() {
-    local searchdirs=("$PKGDEST" "$PWD") results=()
-    local pkg="$1"
-    local search=${pkg%.*}
-    for dir in "${searchdirs[@]}"; do
-        [[ -d $dir ]] || continue
-        results+=$(find "$dir" -type f -name "$search.*" ! -path '*.sig')
-    done
-    case ${#results[*]} in
-        0)
-            return 1
-        ;;
-        1)
-            printf '%s\n' "${results[0]}"
-            return 0
-        ;;
-        *)
-            error 'Multiple packages found:'
-            printf '\t%s\n' "${results[@]}" >&2
-            return 1
-        ;;
-    esac
-}
-
 get_pkgbasename(){
     local pkg="$1"
     local pkgbasename name ver rel arch
 
-    pkgbasename=${pkg%.pkg.tar.?z}
+    pkgbasename=${pkg%.pkg.tar*}
     arch=${pkgbasename##*-}
     pkgbasename=${pkgbasename%-"$arch"}
 
@@ -57,86 +33,42 @@ get_pkgbasename(){
     echo $name
 }
 
-update_repo2(){
-    local repo="$1"
-    local repo_path=${REPOS_ROOT}/$repo/os/${ARCH} packages=()
-
-    for name in ${passfiles[@]}; do
-        if pkgfile=$(find_cached_pkgfile "$name");then
-            info "Found: %s" "$name"
-            if ${add_pkg};then
-                local action='add'
-                packages+=("$name")
-                if ${sign_pkg};then
-                    [[ -e ${pkgfile}.sig ]] && rm ${pkgfile}.sig
-                    signfile ${pkgfile}
+add(){
+    packages+=("$pkg")
+    if [[ -e "${pkgfile}" ]]; then
+        action='add'
+        ln -sf "${pkgfile}"{,.sig} "$repo_path"/
     fi
-                ln -sf ${pkgfile}{,.sig} $repo_path/
-            elif ${del_pkg};then
-                local action='remove'
-                packages+=("$(get_pkgbasename "$name")")
-                [[ -e $repo_path/$name ]] && rm $repo_path/$name
-                [[ -e $repo_path/$name.sig ]] && rm $repo_path/$name.sig
-            fi
-        fi
-    done
-    cd $repo_path
-    if [[ -n $action ]]; then
-        repo-$action -R $repo.${PKGDBEXT} ${packages[@]}
-        ${linksdb} && links-$action $repo.${LINKSDBEXT} ${packages[@]}
-    fi
-
-    return 0
 }
 
-update_repo(){
-    local repo="$1" pkgfile ver
-    local repo_path=${REPOS_ROOT}/$repo/os/${ARCH} packages=()
-
-    . PKGBUILD
-
-    local pkgsearch=(${pkgname[@]})
-    if check_option "debug" "y"; then
-        pkgbase=${pkgbase:-${pkgname[@]}}
-        pkgsearch+=("${pkgbase}-debug")
+remove(){
+    packages+=("$(get_pkgbasename "$pkg")")
+    if [[ -e "$repo_path"/"$pkg" ]]; then
+        action='remove'
+        rm "$repo_path"/"$pkg"
     fi
+    [[ -e "$repo_path"/"$pkg".sig ]] && rm "$repo_path"/"$pkg".sig
+}
 
-    for name in ${pkgsearch[@]}; do
-        pkgarch=$(get_pkg_arch "$name")
-        ver=$(get_full_version)
-        if pkgfile=$(find_cached_package "$name" "$ver" "$pkgarch");then
-            local pkg=${pkgfile##*/}
-            info "Found: %s" "$pkg"
-            if ${add_pkg};then
-                local action='add'
-                packages+=("$pkg")
-                if ${sign_pkg};then
-                    [[ -e ${pkgfile}.sig ]] && rm ${pkgfile}.sig
-                    signfile ${pkgfile}
-                fi
-                ln -sf ${pkgfile}{,.sig} $repo_path/
-            elif ${del_pkg};then
-                local action='remove'
-                packages+=("$name")
-                [[ -e $repo_path/$pkg ]] && rm $repo_path/$pkg
-                [[ -e $repo_path/$pkg.sig ]] && rm $repo_path/$pkg.sig
-            fi
+repo_action(){
+    local packages=() action= func="$1"
+    for pkg in ${passfiles[@]}; do
+        if pkgfile=$(find_cached_pkgfile "$pkg");then
+            msg "Found: %s" "${pkgfile}"
+            "$func"
         fi
     done
     cd $repo_path
-    if [[ -n $action ]]; then
-        repo-$action -R $repo.${PKGDBEXT} ${packages[@]}
-        ${linksdb} && links-$action $repo.${LINKSDBEXT} ${packages[@]}
+    if [[ -n "$action" ]]; then
+        repo-"$action" -R "${dest_repo}"."${PKGDBEXT}" "${packages[@]}"
+        ${linksdb} && links-"$action" "${dest_repo}"."${LINKSDBEXT}" "${packages[@]}"
     fi
-
-    return 0
 }
 
 load_makepkg_config
 
 add_pkg=false
-del_pkg=false
-sign_pkg=false
+rm_pkg=false
 linksdb=false
 
 cmd=${0##*/}
@@ -144,24 +76,24 @@ dest_repo=${cmd#*-}
 
 usage() {
     echo "Usage: ${cmd} [options]"
+    echo '    -d <dest>          Destination repository'
     echo '    -a                 Add package(s) to repository'
     echo '    -r                 Remove package(s) from repository'
     echo '    -l                 Use links db'
-    echo '    -s                 Sign package(s)'
     echo '    -h                 This help'
     echo ''
     echo ''
     exit $1
 }
 
-opts='arlsh'
+opts='arlhd:'
 
 while getopts "${opts}" arg; do
     case "${arg}" in
-        a) add_pkg=true; del_pkg=false ;;
-        r) del_pkg=true; add_pkg=false ;;
+        d) dest_repo="$OPTARG" ;;
+        a) add_pkg=true; rm_pkg=false ;;
+        r) rm_pkg=true; add_pkg=false ;;
         l) linksdb=true ;;
-        s) sign_pkg=true ;;
         h|?) usage 0 ;;
         *) echo "invalid argument '${arg}'"; usage 1 ;;
     esac
@@ -173,8 +105,13 @@ passfiles="$@"
 
 prepare_dir "${REPOS_ROOT}"
 
+repo_path=${REPOS_ROOT}/${dest_repo}/os/${ARCH}
+
 if [[ -n ${passfiles[@]} ]]; then
-    update_repo2 "${dest_repo}"
-else
-    update_repo "${dest_repo}"
+    if ${add_pkg}; then
+        repo_action add
+    fi
+    if ${rm_pkg}; then
+        repo_action remove
+    fi
 fi

bin/pkg/signpkg.in

@@ -0,0 +1,46 @@
+#!/bin/bash
+#
+# Copyright (C) 2018-19 artoo@artixlinux.org
+# Copyright (C) 2018 Artix Linux Developers
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+. @libdir@/artools/util-pkg.sh
+
+prepare_artools
+
+load_makepkg_config
+
+passfiles="$@"
+
+sign_pkg(){
+    local file_to_sign="$1"
+
+    if [ ! -e "$file_to_sign" ]; then
+        error "%s does not exist!" "$file_to_sign"
+        exit 1
+    fi
+
+    if [[ -n "${BUILDBOT_GPGP}" ]]; then
+        msg2 "Signing [%s]" "${file_to_sign##*/}"
+        gpg --batch --passphrase "${BUILDBOT_GPGP}" --detach-sign "$file_to_sign"
+    else
+        msg2 "Signing [%s] with key %s" "${file_to_sign##*/}" "${GPGKEY}..."
+        gpg --detach-sign --use-agent -u "${GPGKEY}" "$file_to_sign"
+    fi
+}
+
+for pkg in ${passfiles[@]}; do
+    if pkgfile=$(find_cached_pkgfile "$pkg");then
+        msg "Found: %s" "${pkgfile}"
+        [[ -e "${pkgfile}".sig ]] && rm "${pkgfile}".sig
+        sign_pkg "${pkgfile}"
+    fi
+done

data/conf/artools-pkg.conf

@@ -15,6 +15,7 @@
 #         community
 #         packages-kernel
 #         packages-net
+#         packages-gfx
 #         packages-openrc
 #         packages-runit
 #         packages-s6

lib/iso/chrootcfg.sh

@@ -36,7 +36,9 @@ add_svc_runit(){
 add_svc_s6(){
     local mnt="$1" names="$2" valid="" rlvl="${3:-default}"
     for svc in $names; do
-        if [[ -d $mnt/etc/s6/sv/$svc ]]; then
+        error=false
+        chroot $mnt s6-rc-db -c /etc/s6/rc/compiled type $svc &> /dev/null || error=true
+        if [ $? == 0 ] && [[ $error == false ]]; then
             msg2 "Setting %s ..." "$svc"
             valid=${valid:-}${valid:+' '}${svc}
         fi

lib/pkg/subrepo.sh

@@ -33,7 +33,7 @@ subrepo_init() {
 subrepo_push(){
     local pkg="$1"
     msg2 "Subrepo push (%s)" "$pkg"
-    git subrepo push "$pkg"
+    git subrepo push "$pkg" || die "%s push failed" "$pkg"
 }
 
 subrepo_clean(){

lib/pkg/util.sh

@@ -16,10 +16,10 @@ shopt -s extglob
 
 get_compliant_name(){
     local gitname="$1"
-    case $gitname in
+    case "$gitname" in
         *+) gitname=${gitname//+/plus}
     esac
-    echo $gitname
+    echo "$gitname"
 }
 
 set_arch_repos(){
@@ -42,56 +42,17 @@ find_repo(){
 
 find_pkg(){
     local searchdir="$1" pkg="$2"
-    local result=$(find $searchdir -mindepth 2 -maxdepth 2 -type d -name "$pkg")
-    echo $result
+    local result=$(find "$searchdir" -mindepth 2 -maxdepth 2 -type d -name "$pkg")
+    echo "$result"
 }
 
-pkgver_equal() {
-    if [[ $1 = *-* && $2 = *-* ]]; then
-        # if both versions have a pkgrel, then they must be an exact match
-        [[ $1 = "$2" ]]
-    else
-        # otherwise, trim any pkgrel and compare the bare version.
-        [[ ${1%%-*} = "${2%%-*}" ]]
-    fi
-}
-
-find_cached_package() {
+find_cached_pkgfile() {
     local searchdirs=("$PKGDEST" "$PWD") results=()
-    local targetname=$1 targetver=$2 targetarch=$3
-    local dir pkg pkgbasename name ver rel arch r results
-
+    local pkg="$1"
     for dir in "${searchdirs[@]}"; do
-        [[ -d $dir ]] || continue
-
-        for pkg in "$dir"/*.pkg.tar?(.!(sig|*.*)); do
-            [[ -f $pkg ]] || continue
-
-            # avoid adding duplicates of the same inode
-            for r in "${results[@]}"; do
-                [[ $r -ef $pkg ]] && continue 2
+        [[ -d "$dir" ]] || continue
+        [[ -e "$dir/$pkg" ]] && results+=("$dir/$pkg")
     done
-
-            # split apart package filename into parts
-            pkgbasename=${pkg##*/}
-            pkgbasename=${pkgbasename%.pkg.tar*}
-
-            arch=${pkgbasename##*-}
-            pkgbasename=${pkgbasename%-"$arch"}
-
-            rel=${pkgbasename##*-}
-            pkgbasename=${pkgbasename%-"$rel"}
-
-            ver=${pkgbasename##*-}
-            name=${pkgbasename%-"$ver"}
-
-            if [[ $targetname = "$name" && $targetarch = "$arch" ]] &&
-                pkgver_equal "$targetver" "$ver-$rel"; then
-                results+=("$pkg")
-            fi
-        done
-    done
-
     case ${#results[*]} in
         0)
             return 1

lib/util-pkg.sh.in

@@ -42,6 +42,7 @@ load_pkg_config(){
             community
             packages-kernel
             packages-net
+            packages-gfx
             packages-openrc
             packages-runit
             packages-s6