4.0.7: bump dependencies, minor changes

set up private api correctly and only send irc messages if enabled in config

Dockerfile

@@ -1,4 +1,8 @@
-FROM artixlinux/artixlinux:base-devel as build-env
+FROM artixlinux/artixlinux:base-devel AS baseimg
+
+RUN pacman -Syu --noconfirm
+
+FROM baseimg AS build-env
 
 WORKDIR /usr/notifier
 
@@ -14,12 +18,12 @@ RUN npm run-script build && \
   npm ci --only=production
 
 
-FROM artixlinux/artixlinux:base-devel as deploy
+FROM baseimg AS deploy
 
 VOLUME /usr/notifier/config
 WORKDIR /usr/notifier
 HEALTHCHECK  --timeout=15m \
-  CMD curl --fail http://localhost:8080/healthcheck || exit 1
+  CMD curl --fail http://localhost:8081/healthcheck || exit 1
 
 EXPOSE 8080
 
@@ -38,8 +42,8 @@ RUN mkdir -p ./config /home/artix/.config/artix-checkupdates \
 
 USER artix
 
-ENV ARTIX_MIRROR="https://mirrors.qontinuum.space/artixlinux/%s/os/x86_64"
-ENV ARCH_MIRROR="https://mirrors.qontinuum.space/archlinux/%s/os/x86_64"
+ENV ARTIX_MIRROR="https://mirror.sanin.dev/artix-linux/%s/os/x86_64/"
+ENV ARCH_MIRROR="https://mirror.sanin.dev/arch-linux/%s/os/x86_64/"
 ENV ARTIX_REPOS="system-goblins,world-goblins,galaxy-goblins,lib32-goblins,system-gremlins,world-gremlins,galaxy-gremlins,lib32-gremlins,system,world,galaxy,lib32"
 ENV ARCH_REPOS="core-staging,extra-staging,multilib-staging,core-testing,extra-testing,multilib-testing,core,extra,multilib"
 ENV GITEA_TOKEN="CHANGEME"

daemon.js

@@ -0,0 +1,309 @@
+const path = require('path');
+const fs = require('fs');
+const os = require('os');
+const spawn = require('child_process').spawn;
+const cron = require('node-cron');
+const dayjs = require('dayjs');
+const express = require('express');
+const phin = require('phin');
+const DB = require('./db');
+const fsp = fs.promises;
+
+const TIMEOUT = 180000;
+const ORPHAN = {
+    "name": "orphan",
+    "ircName": "orphaned"
+};
+const EXTRASPACE = new RegExp('\\s+', 'g');
+const CHECKUPDATESCACHE = path.join(os.homedir(), '.cache', 'artix-checkupdates');
+const NICETYPES = {
+    move: 'move',
+    udate: 'update'
+}
+const NAMECOMPLIANCE = [
+    p => p.replace(/([a-zA-Z0-9]+)\+([a-zA-Z]+)/g, '$1-$2'),
+    p => p.replace(/\+/g, "plus"),
+    p => p.replace(/[^a-zA-Z0-9_\-\.]/g, "-"),
+    p => p.replace(/[_\-]{2,}/g, "-")
+]
+
+class Daemon {
+
+    constructor(config) {
+        const app = express();
+        const port = process.env.PRIVATEPORT || config.privateport || 8081;
+        this._config = config;
+        this._savePath = process.env.SAVEPATH || config.savePath || path.join(__dirname, 'config', 'data.json');
+        console.log('Written by Cory Sanin for Artix Linux');
+        this._locked = false;
+        const db = this._db = new DB(process.env.DBPATH || config.db || path.join(__dirname, 'config', 'packages.db'));
+        this._saveData = {
+            'last-sync': null,
+            move: [],
+            update: []
+        };
+
+        app.set('trust proxy', 1);
+
+        app.get('/healthcheck', (req, res) => {
+            res.send('Healthy');
+        });
+
+        this.readSaveData();
+
+        // resetting flags in case of improper shutdown
+        db.restoreFlags();
+
+        this._cronjob = cron.schedule(process.env.CRON || config.cron || '*/30 * * * *', () => {
+            this.main(this._config);
+        });
+
+        this._webserver = app.listen(port);
+    }
+
+    main = async (config) => {
+        const db = this._db;
+        if (this._locked) {
+            return
+        }
+        this._locked = true;
+        console.log('Starting scheduled task');
+        let now = dayjs();
+        if (!('last-sync' in this._saveData) || !this._saveData['last-sync'] || dayjs(this._saveData['last-sync']).isBefore(now.subtract(process.env.SYNCFREQ || config.syncfreq || 2, 'days'))) {
+            await this.updateMaintainers(config);
+            this._saveData['last-sync'] = now.toJSON();
+            await this.writeSaveData();
+        }
+        await this.checkupdates(config);
+        await this.writeSaveData();
+        this._locked = false;
+        console.log('Task complete.');
+    }
+
+    cleanUpLockfiles = async () => {
+        try {
+            await fsp.rm(CHECKUPDATESCACHE, { recursive: true, force: true });
+        }
+        catch (ex) {
+            console.error('Failed to remove the artix-checkupdates cache directory:', ex);
+        }
+    }
+
+    updateMaintainers = async (config) => {
+        const db = this._db;
+        console.log('Syncing packages...');
+        const lastseen = (new Date()).getTime();
+        const maintainers = [...(config.maintainers || []), ORPHAN];
+        for (let i = 0; i < maintainers.length; i++) {
+            let maintainer = maintainers[i];
+            if (typeof maintainer === 'object') {
+                maintainer = maintainer.name;
+            }
+            console.log(`Syncing ${maintainer}...`);
+            try {
+                const packages = await this.getMaintainersPackages(maintainer);
+                for (let j = 0; j < packages.length; j++) {
+                    db.updatePackage(packages[j], maintainer, lastseen);
+                }
+            }
+            catch (err) {
+                console.error(`Failed to get packages for ${maintainer}`);
+                console.error(err);
+            }
+        }
+        console.log(`removing unused packages...`);
+        db.cleanOldPackages(lastseen);
+        console.log(`Package sync complete`);
+    }
+
+    checkupdates = async (config) => {
+        const db = this._db;
+
+        try {
+            await this.handleUpdates(config, this._saveData.move = await this.execCheckUpdates(['-m']), 'move');
+            await this.handleUpdates(config, this._saveData.update = await this.execCheckUpdates(['-u']), 'udate');
+        }
+        catch (ex) {
+            console.error('Failed to check for updates:', ex);
+        }
+    }
+
+    execCheckUpdates = (flags) => {
+        return new Promise((resolve, reject) => {
+            let process = spawn('artix-checkupdates', flags);
+            let timeout = setTimeout(async () => {
+                process.kill() && await this.cleanUpLockfiles();
+                reject('Timed out');
+            }, TIMEOUT);
+            let outputstr = '';
+            let errorOutput = '';
+            process.stdout.on('data', data => {
+                outputstr += data.toString();
+            });
+            process.stderr.on('data', err => {
+                const errstr = err.toString();
+                errorOutput += `${errstr}, `;
+                console.error(errstr);
+            });
+            process.on('exit', async (code) => {
+                if (code === 0 && errorOutput.length === 0) {
+                    clearTimeout(timeout);
+                    resolve(this.parseCheckUpdatesOutput(outputstr));
+                }
+                else {
+                    errorOutput.includes('unable to lock database') && this.cleanUpLockfiles();
+                    reject((code && `exited with ${code}`) || errorOutput);
+                }
+            });
+        });
+    }
+
+    getMaintainersPackages = (maintainer) => {
+        return new Promise((resolve, reject) => {
+            let process = spawn('artixpkg', ['admin', 'query', maintainer === ORPHAN.name ? '-t' : '-m', maintainer]);
+            let timeout = setTimeout(() => {
+                reject('Timed out');
+                process.kill();
+            }, TIMEOUT);
+            let packagelist = [];
+            process.stdout.on('data', data => {
+                packagelist = packagelist.concat(data.toString().trim().split('\n'));
+            });
+            process.stderr.on('data', err => {
+                console.error(err.toString());
+            })
+            process.on('exit', async (code) => {
+                if (code === 0) {
+                    clearTimeout(timeout);
+                    resolve(packagelist);
+                }
+                else {
+                    reject(code);
+                }
+            });
+        });
+    }
+
+    parseCheckUpdatesOutput = (output) => {
+        let packages = [];
+        const lines = output.split('\n');
+        lines.forEach(l => {
+            // "package" is "reserved"
+            const reservethis = l.trim().replace(EXTRASPACE, ' ');
+            if (reservethis.length > 0 && reservethis.indexOf('Package basename') < 0) {
+                packages.push(NAMECOMPLIANCE.reduce((s, fn) => fn(s), reservethis.split(' ', 2)[0]));
+            }
+        });
+        return packages;
+    }
+
+    handleUpdates = async (config, packs, type) => {
+        const db = this._db;
+        packs.forEach(v => {
+            let p = db.getPackage(v);
+            p && db.updateFlag(v, type, p[type] > 0 ? 2 : 4);
+        });
+        const maintainers = [...config.maintainers, ORPHAN];
+        for (let i = 0; i < maintainers.length; i++) {
+            const m = maintainers[i];
+            const mname = typeof m === 'object' ? m.name : m;
+            const ircName = typeof m === 'object' ? (m.ircName || mname) : m;
+            const packages = db.getNewByMaintainer(mname, type);
+            if (typeof m === 'object' && m.channels) {
+                this.notify({
+                    api: config.apprise,
+                    urls: m.channels
+                }, packages, NICETYPES[type]);
+            }
+            this.ircNotify(packages, ircName, NICETYPES[type]);
+        }
+
+        db.decrementFlags(type);
+        db.restoreFlags(type);
+    }
+
+    notify = async (apprise, packarr, type) => {
+        if (!(packarr && packarr.length && apprise && apprise.api && apprise.urls)) {
+            return;
+        }
+        const packagesStr = packarr.map(p => p.package).join('\n');
+        for (let i = 0; i < 25; i++) {
+            try {
+                return await phin({
+                    url: `${apprise.api}/notify/`,
+                    method: 'POST',
+                    data: {
+                        title: `${packarr[0].maintainer}: packages ready to ${type}`,
+                        body: packagesStr,
+                        urls: apprise.urls.join(',')
+                    }
+                });
+            }
+            catch (ex) {
+                console.error('Failed to send notification, attempt #%d', i + 1);
+                console.error(ex);
+            }
+        }
+        return null;
+    }
+
+    ircNotify = async (packarr, maintainer, type) => {
+        const config = this._config;
+        if (!(packarr && packarr.length && config['irc-framework'])) {
+            return;
+        }
+        const hostname = process.env.IRCHOSTNAME || config.irchostname || 'http://artix-notifier-irc:8081';
+        const packagesStr = packarr.map(p => p.package).join('\n');
+        for (let i = 0; i < 25; i++) {
+            try {
+                return await phin({
+                    url: `${hostname}/api/1.0/notifications`,
+                    method: 'POST',
+                    data: {
+                        message: `${maintainer}: packages ready to ${type}\n${packagesStr}\n-------- EOF --------`
+                    }
+                });
+            }
+            catch (ex) {
+                console.error('Failed to send IRC notification, attempt #%d', i + 1);
+                console.error(ex);
+            }
+        }
+        return null;
+    }
+
+    readSaveData = async () => {
+        try {
+            this._saveData = JSON.parse(await fsp.readFile(this._savePath));
+        }
+        catch {
+            console.error(`Failed to read existing save data at ${this._savePath}`);
+        }
+    }
+
+    writeSaveData = async () => {
+        const config = this._config;
+        const hostname = process.env.WEBHOSTNAME || config.webhostname || 'http://artix-notifier-web:8081';
+        try {
+            await fsp.writeFile(this._savePath, JSON.stringify(this._saveData));
+            phin({
+                url: `${hostname}/api/1.0/data`,
+                method: 'PUT'
+            });
+        }
+        catch {
+            console.error(`Failed to write save data to ${this._savePath}`);
+        }
+    }
+
+    close = () => {
+        if (this._webserver) {
+            this._webserver.close();
+        }
+        if (this._cronjob) {
+            this._cronjob.stop();
+        }
+    }
+}
+
+module.exports = Daemon;

db.js

@@ -5,6 +5,7 @@ class DB {
     constructor(file) {
         this._db = new sqlite(file);
         const db = this._db;
+        db.pragma('journal_mode = WAL');
 
         if (!db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='${TABLE}';`).get()) {
             db.prepare(`CREATE TABLE ${TABLE} (package VARCHAR(128) PRIMARY KEY, maintainer VARCHAR(128), move INTEGER, udate INTEGER, lastseen DATETIME);`).run();
@@ -13,6 +14,8 @@ class DB {
         this._queries = {
             GETPACKAGE: db.prepare(`SELECT * FROM ${TABLE} WHERE package = @package;`),
             ADDPACKAGE: db.prepare(`INSERT INTO ${TABLE} (package,maintainer,move,udate,lastseen) VALUES (@package, @maintainer, 0, 0, @lastseen);`),
+            GETPACKAGES: db.prepare(`SELECT package FROM ${TABLE}`),
+            GETPACKAGESSTARTWITH: db.prepare(`SELECT package FROM ${TABLE} WHERE package LIKE @startsWith || '%'`),
             UPDATEMAINTAINER: db.prepare(`UPDATE ${TABLE} SET maintainer = @maintainer, lastseen= @lastseen WHERE package = @package`),
             GETMAINTAINERPACKAGECOUNT: db.prepare(`SELECT COUNT(package) as count FROM ${TABLE} WHERE maintainer = @maintainer;`),
             REMOVEOLDPACKAGE: db.prepare(`DELETE FROM ${TABLE} WHERE lastseen < @lastseen;`),
@@ -23,7 +26,7 @@ class DB {
                 INCREMENT: db.prepare(`UPDATE ${TABLE} SET move = move + 1 WHERE package = @package`),
                 DECREMENT: db.prepare(`UPDATE ${TABLE} SET move = 0 WHERE move = 1`),
                 FIXFLAG: db.prepare(`UPDATE ${TABLE} SET move = 1 WHERE move > 1`),
-                GETPACKAGESBYMAINTAINER: db.prepare(`SELECT * FROM ${TABLE} WHERE maintainer = @maintainer AND move > 0;`),
+                GETPACKAGESBYMAINTAINER: db.prepare(`SELECT * FROM ${TABLE} WHERE maintainer = @maintainer AND move > 0 ORDER BY package ASC;`),
                 GETPACKAGECOUNTBYMAINTAINER: db.prepare(`SELECT COUNT(package) as count FROM ${TABLE} WHERE maintainer = @maintainer AND move > 0;`)
             },
             udate: {
@@ -33,7 +36,7 @@ class DB {
                 INCREMENT: db.prepare(`UPDATE ${TABLE} SET udate = udate + 1 WHERE package = @package`),
                 DECREMENT: db.prepare(`UPDATE ${TABLE} SET udate = 0 WHERE udate = 1`),
                 FIXFLAG: db.prepare(`UPDATE ${TABLE} SET udate = 1 WHERE udate > 1`),
-                GETPACKAGESBYMAINTAINER: db.prepare(`SELECT * FROM ${TABLE} WHERE maintainer = @maintainer AND udate > 0;`),
+                GETPACKAGESBYMAINTAINER: db.prepare(`SELECT * FROM ${TABLE} WHERE maintainer = @maintainer AND udate > 0 ORDER BY package ASC;`),
                 GETPACKAGECOUNTBYMAINTAINER: db.prepare(`SELECT COUNT(package) as count FROM ${TABLE} WHERE maintainer = @maintainer AND udate > 0;`)
             }
         };
@@ -50,6 +53,11 @@ class DB {
         });
     }
 
+    getPackages(startsWith = null) {
+        return ((!!startsWith) ? this._queries.GETPACKAGESSTARTWITH.all({ startsWith }) :
+            this._queries.GETPACKAGES.all()).map(p => p.package);
+    }
+
     updatePackage(pack, maintainer, lastseen) {
         return this._queries[this.getPackage(pack) ? 'UPDATEMAINTAINER' : 'ADDPACKAGE'].run({
             package: pack,

docker-compose.yml

@@ -1,7 +1,31 @@
 version: '2'
 
 services:
-   packy:
+   artix-notifier-daemon:
+      container_name: artix-notifier-daemon
+      build:
+         context: ./
+      volumes:
+         - ./config:/usr/notifier/config
+      depends_on:
+         - artix-notifier-irc
+         - artix-notifier-web
+      environment:
+         COMPONENT: "daemon"
+         ARTIX_REPOS: "system-goblins,world-goblins,system-gremlins,world-gremlins,system,world"
+         ARCH_REPOS: "core-staging,extra-staging,core-testing,extra-testing,core,extra"
+
+   artix-notifier-irc:
+      container_name: artix-notifier-irc
+      build:
+         context: ./
+      volumes:
+         - ./config:/usr/notifier/config
+      environment:
+         COMPONENT: "ircbot"
+
+   artix-notifier-web:
+      container_name: artix-notifier-web
       build:
          context: ./
       volumes:
@@ -9,7 +33,4 @@ services:
       ports:
          - 8080:8080
       environment:
-         ARTIX_MIRROR: "https://mirrors.qontinuum.space/artixlinux/%s/os/x86_64"
-         ARCH_MIRROR: "https://mirrors.qontinuum.space/archlinux/%s/os/x86_64"
-         ARTIX_REPOS: "system-goblins,world-goblins,system-gremlins,world-gremlins,system,world"
-         ARCH_REPOS: "core-staging,extra-staging,core-testing,extra-testing,core,extra"
+         COMPONENT: "web"

index.js

@@ -1,288 +1,33 @@
 const path = require('path');
 const fs = require('fs');
-const os = require('os');
-const spawn = require('child_process').spawn;
-const cron = require('node-cron');
-const dayjs = require('dayjs');
 const json5 = require('json5');
-const phin = require('phin');
-const DB = require('./db');
-const IRCBot = require('./ircbot');
-const Web = require('./web');
-const fsp = fs.promises;
 
-const TIMEOUT = 180000;
-const ORPHAN = {
-    "name": "orphan",
-    "ircName": "orphaned"
-};
-const EXTRASPACE = new RegExp('\\s+', 'g');
-const CHECKUPDATESCACHE = path.join(os.homedir(), '.cache', 'artix-checkupdates');
-const NICETYPES = {
-    move: 'move',
-    udate: 'update'
-}
-
-let saveData = {
-    'last-sync': null,
-    move: [],
-    update: []
-}
-
-let ircBot;
-let locked = false;
-
-let savePath = process.env.SAVEPATH || path.join(__dirname, 'config', 'data.json');
-
-fs.readFile(process.env.CONFIGPATH || path.join(__dirname, 'config', 'config.json'), async (err, data) => {
+fs.readFile(process.env.CONFIG || path.join(__dirname, 'config', 'config.json'), (err, data) => {
     if (err) {
         console.error(err);
-        process.exit(1);
     }
     else {
-        console.log('Written by Cory Sanin for Artix Linux');
-        const config = json5.parse(data);
-        savePath = config.savePath || savePath;
-        const db = new DB(process.env.DBPATH || config.db || path.join(__dirname, 'config', 'packages.db'));
-
-        try {
-            saveData = JSON.parse(await fsp.readFile(savePath));
+        let config = json5.parse(data);
+        let arg = process.env.COMPONENT || (process.execArgv && process.execArgv[0]);
+        
+        if (arg === 'daemon') {
+            const Daemon = require('./daemon');
+            const daemon = new Daemon(config);
+            process.on('SIGTERM', daemon.close);
         }
-        catch {
-            console.error(`Failed to read existing save data at ${savePath}`);
+        else if (arg === 'ircbot') {
+            const IRCBot = require('./ircbot');
+            const bot = new IRCBot(config);
+            bot.connect();
+            process.on('SIGTERM', bot.close);
         }
-
-        // resetting flags in case of improper shutdown
-        db.restoreFlags();
-
-        let cronjob = cron.schedule(process.env.CRON || config.cron || '*/30 * * * *', () => {
-            main(config, db);
-        });
-
-        const web = new Web(db, config, saveData);
-        ircBot = new IRCBot(config);
-        ircBot.connect();
-
-        process.on('SIGTERM', () => {
-            cronjob.stop();
-            web.close();
-            ircBot.close();
-        });
-    }
-});
-
-async function main(config, db) {
-    if (locked) {
-        return
-    }
-    locked = true;
-    console.log('Starting scheduled task');
-    let now = dayjs();
-    if (!('last-sync' in saveData) || !saveData['last-sync'] || dayjs(saveData['last-sync']).isBefore(now.subtract(process.env.SYNCFREQ || config.syncfreq || 2, 'days'))) {
-        ircBot.close();
-        await updateMaintainers(config, db);
-        saveData['last-sync'] = now.toJSON();
-        await writeSaveData();
-        await ircBot.connect();
-    }
-    await checkupdates(config, db);
-    await writeSaveData();
-    locked = false;
-    console.log('Task complete.');
-}
-
-async function writeSaveData() {
-    try {
-        await fsp.writeFile(savePath, JSON.stringify(saveData));
-    }
-    catch {
-        console.error(`Failed to write save data to ${savePath}`);
-    }
-}
-
-async function checkupdates(config, db) {
-    try {
-        await handleUpdates(config, db, saveData.move = await execCheckUpdates(['-m']), 'move');
-        await handleUpdates(config, db, saveData.update = await execCheckUpdates(['-u']), 'udate');
-    }
-    catch (ex) {
-        console.error('Failed to check for updates:', ex);
-    }
-}
-
-async function handleUpdates(config, db, packs, type) {
-    packs.forEach(v => {
-        let p = db.getPackage(v);
-        p && db.updateFlag(v, type, p[type] > 0 ? 2 : 4);
-    });
-    const maintainers = [...config.maintainers, ORPHAN];
-    for (let i = 0; i < maintainers.length; i++) {
-        const m = maintainers[i];
-        const mname = typeof m === 'object' ? m.name : m;
-        const ircName = typeof m === 'object' ? (m.ircName || mname) : m;
-        const packages = db.getNewByMaintainer(mname, type);
-        if (typeof m === 'object' && m.channels) {
-            notify({
-                api: config.apprise,
-                urls: m.channels
-            }, packages, NICETYPES[type])
+        else if (arg === 'web') {
+            const Web = require('./web');
+            const web = new Web(config);
+            process.on('SIGTERM', web.close);
         }
-        ircNotify(packages, ircName, NICETYPES[type]);
-    }
-
-    db.decrementFlags(type);
-    db.restoreFlags(type);
-}
-
-async function updateMaintainers(config, db) {
-    console.log('Syncing packages...');
-    const lastseen = (new Date()).getTime();
-    const maintainers = [...(config.maintainers || []), ORPHAN];
-    for (let i = 0; i < maintainers.length; i++) {
-        let maintainer = maintainers[i];
-        if (typeof maintainer === 'object') {
-            maintainer = maintainer.name;
-        }
-        console.log(`Syncing ${maintainer}...`);
-        try {
-            const packages = await getMaintainersPackages(maintainer);
-            for (let j = 0; j < packages.length; j++) {
-                const package = packages[j];
-                db.updatePackage(package, maintainer, lastseen);
-                await asyncSleep(50);
-            }
-        }
-        catch (err) {
-            console.error(`Failed to get packages for ${maintainer}`);
-            console.error(err);
+        else {
+            console.error('Please pass the component you wish to run.');
         }
     }
-    console.log(`removing unused packages...`);
-    db.cleanOldPackages(lastseen);
-    console.log(`Package sync complete`);
-}
-
-function getMaintainersPackages(maintainer) {
-    return new Promise((resolve, reject) => {
-        let process = spawn('artixpkg', ['admin', 'query', maintainer === ORPHAN.name ? '-t' : '-m', maintainer]);
-        let timeout = setTimeout(() => {
-            reject('Timed out');
-            process.kill();
-        }, TIMEOUT);
-        let packagelist = [];
-        process.stdout.on('data', data => {
-            packagelist = packagelist.concat(data.toString().trim().split('\n'));
-        });
-        process.stderr.on('data', err => {
-            console.error(err.toString());
-        })
-        process.on('exit', async (code) => {
-            if (code === 0) {
-                clearTimeout(timeout);
-                resolve(packagelist);
-            }
-            else {
-                reject(code);
-            }
-        });
-    });
-}
-
-async function notify(apprise, packarr, type) {
-    if (!(packarr && packarr.length && apprise && apprise.api && apprise.urls)) {
-        return;
-    }
-    const packagesStr = packarr.map(p => p.package).join('\n');
-    for (let i = 0; i < 25; i++) {
-        try {
-            return await phin({
-                url: `${apprise.api}/notify/`,
-                method: 'POST',
-                data: {
-                    title: `${packarr[0].maintainer}: packages ready to ${type}`,
-                    body: packagesStr,
-                    urls: apprise.urls.join(',')
-                }
-            });
-        }
-        catch (ex) {
-            console.error('Failed to send notification, attempt #%d', i + 1);
-            console.error(ex);
-        }
-    }
-    return null;
-}
-
-function ircNotify(packarr, maintainer, type) {
-    if (!(packarr && packarr.length)) {
-        return;
-    }
-    const packagesStr = packarr.map(p => p.package).join('\n');
-    for (let i = 0; i < 25; i++) {
-        try {
-            return ircBot.sendMessage(`${maintainer}: packages ready to ${type}\n${packagesStr}\n-------- EOF --------`);
-        }
-        catch (ex) {
-            console.error('Failed to send IRC notification, attempt #%d', i + 1);
-            console.error(ex);
-        }
-    }
-    return null;
-}
-
-function parseCheckUpdatesOutput(output) {
-    let packages = [];
-    let lines = output.split('\n');
-    lines.forEach(l => {
-        let package = l.trim().replace(EXTRASPACE, ' ');
-        if (package.length > 0 && package.indexOf('Package basename') < 0) {
-            packages.push(package.split(' ', 2)[0]);
-        }
-    });
-    return packages;
-}
-
-async function cleanUpLockfiles() {
-    try {
-        await fsp.rm(CHECKUPDATESCACHE, { recursive: true, force: true });
-    }
-    catch (ex) {
-        console.error('Failed to remove the artix-checkupdates cache directory:', ex);
-    }
-}
-
-function execCheckUpdates(flags, errCallback) {
-    return new Promise((resolve, reject) => {
-        let process = spawn('artix-checkupdates', flags);
-        let timeout = setTimeout(async () => {
-            process.kill() && await cleanUpLockfiles();
-            reject('Timed out');
-        }, TIMEOUT);
-        let outputstr = '';
-        let errorOutput = '';
-        process.stdout.on('data', data => {
-            outputstr += data.toString();
-        });
-        process.stderr.on('data', err => {
-            const errstr = err.toString();
-            errorOutput += `${errstr}, `;
-            console.error(errstr);
-        })
-        process.on('exit', async (code) => {
-            if (code === 0 && errorOutput.length === 0) {
-                clearTimeout(timeout);
-                resolve(parseCheckUpdatesOutput(outputstr));
-            }
-            else {
-                errorOutput.includes('unable to lock database') && cleanUpLockfiles();
-                reject((code && `exited with ${code}`) || errorOutput);
-            }
-        });
-    });
-}
-
-function asyncSleep(ms) {
-    return new Promise((resolve) => {
-        setTimeout(resolve, ms);
-    });
-}
+});

ircbot.js

@@ -1,13 +1,46 @@
 const IRC = require('irc-framework');
+const express = require('express');
 
 class IRCBot {
 
     constructor(config) {
         const options = config['irc-framework'];
         const aux = this._aux = config.ircClient || {};
+        const app = express();
+        const port = process.env.PRIVATEPORT || config.privateport || 8081;
         this._channel = aux.channel;
         this._messageQueue = [];
         this._enabled = !!options;
+
+        app.set('trust proxy', 1);
+        app.use(express.json());
+
+        app.get('/healthcheck', (req, res) => {
+            if (this._bot && this._bot.connected) {
+                res.send('healthy');
+            }
+            else {
+                res.status(500).send('offline');
+                process.exit(1);
+            }
+        });
+
+        app.post('/api/1.0/notifications', (req, res) => {
+            const body = req.body;
+            if (body && body.message) {
+                this.sendMessage(body.message);
+                res.json({
+                    success: true
+                });
+            }
+            else {
+                res.status(400).json({
+                    success: false,
+                    error: 'must include `message` property in request body'
+                });
+            }
+        });
+
         if (options) {
             const bot = this._bot = new IRC.Client(options);
 
@@ -18,6 +51,8 @@ class IRCBot {
             bot.on('action', d => console.log(`irc:action: ${d.message}`));
 
             setInterval(() => this.processMessageQueue(), 2000);
+
+            this._webserver = app.listen(port, () => console.log(`artix-packy-notifier-irc running on port ${port}`));
         }
         else {
             console.log('"ircClient" not provided in config. IRC notifications will not be delivered.');
@@ -61,6 +96,9 @@ class IRCBot {
     }
 
     close() {
+        if (this._webserver) {
+            this._webserver.close();
+        }
         if (this._enabled && this._bot.connected) {
             this._bot.quit('Shutting down');
         }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "artix-packy-notifier",
-  "version": "3.4.0",
+  "version": "4.0.7",
   "description": "Determine packages that need attention",
   "main": "index.js",
   "scripts": {
@@ -23,20 +23,20 @@
   },
   "homepage": "https://gitlab.com/sanin.dev/artix-packy-notifier#readme",
   "dependencies": {
-    "better-sqlite3": "9.4.3",
-    "dayjs": "1.11.10",
-    "ejs": "3.1.9",
-    "express": "4.19.1",
-    "irc-framework": "4.13.1",
+    "better-sqlite3": "11.7.2",
+    "dayjs": "1.11.13",
+    "ejs": "3.1.10",
+    "express": "4.21.2",
+    "irc-framework": "4.14.0",
     "json5": "2.2.3",
     "node-cron": "3.0.3",
-    "phin": "^3.7.0",
-    "prom-client": "15.1.0",
-    "sharp": "0.33.2"
+    "phin": "3.7.1",
+    "prom-client": "15.1.3",
+    "sharp": "0.33.5"
   },
   "devDependencies": {
     "csso": "5.0.5",
-    "sass": "1.72.0",
-    "uglify-js": "3.17.4"
+    "sass": "1.83.1",
+    "uglify-js": "3.19.3"
   }
 }

styles/01-main.scss

@@ -164,14 +164,14 @@ table tr td:last-child {
     }
 }
 
-@media screen and (max-width:790px) {
+@media screen and (max-width:910px) {
     .container {
         width: initial;
         margin: .5em .5em;
     }
 }
 
-@media screen and (max-width:660px) {
+@media screen and (max-width:740px) {
     header {
         nav {
             float: initial;

views/head.ejs

@@ -9,6 +9,6 @@
     <meta name="apple-mobile-web-app-title" content="Artix Checkupdates">
     <meta name="application-name" content="Artix Checkupdates">
     <meta name="theme-color" content="#212121">
-    <link rel="stylesheet" href="/assets/css/styles.css?v=2">
+    <link rel="stylesheet" href="/assets/css/styles.css?v=3">
     <link rel="shortcut icon" href="/assets/svg/artix_logo.svg">
 </head>

views/header.ejs

@@ -10,6 +10,7 @@
                 <li><a href="https://forum.artixlinux.org/">Forum</a></li>
                 <li><a href="https://wiki.artixlinux.org/">Wiki</a></li>
                 <li><a href="https://gitea.artixlinux.org/explore/repos">Sources</a></li>
+                <li><a href="https://packages.artixlinux.org/">Packages</a></li>
             </ul>
         </nav>
     </header>

views/index.ejs

@@ -11,7 +11,7 @@
             <% }); %>
             <li><a href="/maintainer/orphan">Orphan Packages</a></li>
         </ul>
-        <h2>All Pending</h2>
+        <h2>All Pending (<%= (packages && packages.length) || 0 %> packages)</h2>
         <table>
             <tr>
                 <th>Package</th>

views/maintainer.ejs

@@ -6,7 +6,7 @@
         <% if (maintainer === 'orphan') { %>
         <h1>Orphaned Packages with Pending Operations</h1>
         <p>
-            There are <a href="https://gitea.artixlinux.org/explore/repos?q=maintainer-<%= maintainer %>&topic=1"><%= packagesOwned %> packages</a> without a maintainer.
+            There are <a href="https://gitea.artixlinux.org/explore/repos?q=<%= maintainer %>&topic=1"><%= packagesOwned %> packages</a> without a maintainer.
         </p>
         <% } else { %>
         <h1><%= maintainer %>'s Operations</h1>
@@ -14,6 +14,9 @@
             <%= maintainer %> owns <a href="https://gitea.artixlinux.org/explore/repos?q=maintainer-<%= maintainer %>&topic=1"><%= packagesOwned %> packages</a>.
         </p>
         <% } %>
+        <p>
+            <%= (packages && packages.length) || 0 %> of which require attention.
+        </p>
         <table>
             <tr>
                 <th>Package</th>
@@ -27,4 +30,5 @@
             <% }); %>
         </table>
     </div>
+    <%- include("footer", locals) %>
 </body>

web.js

@@ -3,25 +3,21 @@ const prom = require('prom-client');
 const sharp = require('sharp');
 const fs = require('fs');
 const path = require('path');
+const DB = require('./db');
+const fsp = fs.promises;
 
 const PROJECT_ROOT = __dirname;
 const VIEWOPTIONS = {
     outputFunctionName: 'echo'
 };
-const NAMECOMPLIANCE = [
-    p => p.replace(/([a-zA-Z0-9]+)\+([a-zA-Z]+)/g, '$1-$2'),
-    p => p.replace(/\+/g, "plus"),
-    p => p.replace(/[^a-zA-Z0-9_\-\.]/g, "-"),
-    p => p.replace(/[_\-]{2,}/g, "-")
-]
 
 function inliner(file) {
     return fs.readFileSync(path.join(PROJECT_ROOT, file));
 }
 
 function packageUrl(p) {
-    let packagename = typeof p === 'string' ? p : p.package;
-    return `https://gitea.artixlinux.org/packages/${NAMECOMPLIANCE.reduce((s, fn) => fn(s), packagename)}`;
+    const packagename = typeof p === 'string' ? p : p.package;
+    return `https://gitea.artixlinux.org/packages/${packagename}`;
 }
 
 function prepPackages(arr, action) {
@@ -94,11 +90,19 @@ async function createOutlinedText(string, meta, gravity = 'west') {
 }
 
 class Web {
-    constructor(db, options, savedata) {
+    constructor(options) {
+        const db = new DB(process.env.DBPATH || options.db || path.join(__dirname, 'config', 'packages.db'));
         const app = express();
+        const privateapp = express();
         const port = process.env.PORT || options.port || 8080;
+        const privateport = process.env.PRIVATEPORT || options.privateport || 8081;
         const METRICPREFIX = process.env.METRICPREFIX || 'artixpackages_';
         const maintainers = this._maintainers = (options.maintainers || []).map(m => typeof m === 'object' ? m.name : m).sort();
+        const savePath = process.env.SAVEPATH || options.savePath || path.join(__dirname, 'config', 'data.json');
+        let saveData = {
+            move: [],
+            update: []
+        };
 
         app.set('trust proxy', 1);
         app.set('view engine', 'ejs');
@@ -128,13 +132,17 @@ class Web {
             );
         }
 
+        async function readSave() {
+            saveData = JSON.parse(await fsp.readFile(savePath));
+        }
+
         app.get('/healthcheck', async (_, res) => {
             res.send('Healthy');
         });
 
         app.get('/', async (_, res) => {
-            let packages = prepPackages(savedata.move, 'Move');
-            packages = packages.concat(prepPackages(savedata.update, 'Update'));
+            let packages = prepPackages(saveData.move, 'Move');
+            packages = packages.concat(prepPackages(saveData.update, 'Update'));
             res.render('index',
                 {
                     inliner,
@@ -228,6 +236,7 @@ class Web {
 
         app.get('/api/1.0/maintainers', (req, res) => {
             const acceptHeader = req.headers.accept;
+            res.set('Cache-Control', 'public, max-age=360');
             if (acceptHeader && acceptHeader.includes('application/json')) {
                 res.json({
                     maintainers
@@ -238,6 +247,37 @@ class Web {
             }
         });
 
+        app.get('/api/1.0/packages', (req, res) => {
+            const acceptHeader = req.headers.accept;
+            const startsWith = req.query.startswith;
+            const packages = db.getPackages(startsWith);
+            res.set('Cache-Control', 'public, max-age=360');
+            if (acceptHeader && acceptHeader.includes('application/json')) {
+                res.json({
+                    packages
+                });
+            }
+            else {
+                res.send(packages.join(' '));
+            }
+        });
+
+        privateapp.put('/api/1.0/data', (req, res) => {
+            try {
+                readSave();
+                res.json({
+                    success: true
+                });
+            }
+            catch(ex) {
+                console.error(ex);
+                res.status(500).json({
+                    success: false,
+                    error: 'failed to read save data'
+                });
+            }
+        });
+
         const register = prom.register;
 
         new prom.Gauge({
@@ -258,11 +298,11 @@ class Web {
                 this.set({
                     maintainer: 'any',
                     action: 'move'
-                }, savedata.move.length);
+                }, saveData.move.length);
                 this.set({
                     maintainer: 'any',
                     action: 'update'
-                }, savedata.update.length);
+                }, saveData.update.length);
             }
         });
 
@@ -296,11 +336,17 @@ class Web {
 
         app.use((req, res) => sendError(req, res, 404, 'File not found'));
 
+        privateapp.use('/', app);
+
+        readSave();
+
         this._webserver = app.listen(port, () => console.log(`artix-packy-notifier-web running on port ${port}`));
+        this._privateserver = privateapp.listen(privateport);
     }
 
     close = () => {
         this._webserver.close();
+        this._privateserver.close();
     }
 }