track source

.gitignore

@@ -102,4 +102,6 @@ public
 distribution/*
 mirrorlist
 mirrors*.json
+mirrors.md
+head.md
 .env

README.md

@@ -16,18 +16,22 @@ artix-mlg expects the input file to contain an array `"mirrors"` of objects foll
 
 ```
 interface MirrorProfile {
-    url: string;               // The full pacman mirror url, e.g. https://mirror.example.com/artix-linux/$repo/os/$arch
-    tier: number;              // https://wiki.archlinux.org/title/DeveloperWiki:NewMirrors#2-tier_mirroring_scheme
-    country: string;           // Country name
-    public: boolean;           // Whether the mirror is meant for users
-    active: boolean;           // Whether the mirror is currently in service
-    default: boolean;          // Whether the mirror is considered a default mirror
-    admin_email?: string;      // (optional) Email address of the mirror administrator
-    alternate_email?: string;  // (optional) Alternate email address for the mirror administrator
-    isos?: boolean;            // (optional) Whether the mirror hosts ISOs (automatically determined by artix-mlg)
-    rsync_user?: string;       // (optional) rsync username, if applicable
-    rsync_password?: string;   // (optional) rsync password, if applicable
-    notes?: string;            // (optional) notes to be stored in archweb
+    url: string;                // The full pacman mirror url, e.g. https://mirror.example.com/artix-linux/$repo/os/$arch
+    tier: number;               // https://wiki.archlinux.org/title/DeveloperWiki:NewMirrors#2-tier_mirroring_scheme
+    country: string;            // Country name
+    upstream: string;           // Upstream mirror name (usually the domain)
+    public: boolean;            // Whether the mirror is meant for users
+    active: boolean;            // Whether the mirror is currently in service
+    default: boolean;           // Whether the mirror is considered a default mirror
+    suppress?: boolean;         // Whether to hide from the resulting mirrorlist (for when active but redundant)
+    admin_email?: string;       // (optional) Email address of the mirror administrator
+    alternate_email?: string;   // (optional) Alternate email address for the mirror administrator
+    stable_isos?: string;       // (optional) URL to stable isos, if present
+    weekly_isos?: string;       // (optional) URL to weekly isos, if present
+    rsync_user?: string;        // (optional) rsync username, if applicable
+    rsync_password?: string;    // (optional) rsync password, if applicable
+    notes?: string;             // (optional) notes to be stored in archweb
+    force_mirror_name?: string; // (optional) force a mirror name (defaults to the domain)
 }
 ```
 

package-lock.json

@@ -1,18 +1,19 @@
 {
   "name": "artix-mlg",
-  "version": "0.2.2",
+  "version": "0.2.7",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "artix-mlg",
-      "version": "0.2.2",
+      "version": "0.2.7",
       "license": "MIT",
       "dependencies": {
         "country-code-lookup": "0.1.3",
-        "extract-tld": "1.1.2",
-        "ky": "1.8.2",
-        "tiny-spin": "1.0.2"
+        "extract-tld": "1.1.2"
+      },
+      "bin": {
+        "artix-mlg": "bin/artix-mlg.mjs"
       },
       "devDependencies": {
         "@types/node": "^24.1.0",
@@ -41,24 +42,6 @@
       "integrity": "sha512-2cF2AD1Xz3C9oemTuL7Atmgdc3l5WZgFvh6TxBEoNMvyUvHK/gX928edzdwsUbwmmJ+fbPclrbRHsZoZSXUiAQ==",
       "license": "ISC"
     },
-    "node_modules/ky": {
-      "version": "1.8.2",
-      "resolved": "https://registry.npmjs.org/ky/-/ky-1.8.2.tgz",
-      "integrity": "sha512-XybQJ3d4Ea1kI27DoelE5ZCT3bSJlibYTtQuMsyzKox3TMyayw1asgQdl54WroAm+fIA3ZCr8zXW2RpR7qWVpA==",
-      "license": "MIT",
-      "engines": {
-        "node": ">=18"
-      },
-      "funding": {
-        "url": "https://github.com/sindresorhus/ky?sponsor=1"
-      }
-    },
-    "node_modules/tiny-spin": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/tiny-spin/-/tiny-spin-1.0.2.tgz",
-      "integrity": "sha512-w+LQXNFIrts+pOjuf1/UivYCd4znPiH/c5X8500Qv6n7FpUDaaB5Q1JSuZN3MIj65qQHzoXcKT9QAz3Tg/djFQ==",
-      "license": "MIT"
-    },
     "node_modules/typescript": {
       "version": "5.8.3",
       "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.8.3.tgz",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "artix-mlg",
-  "version": "0.2.2",
+  "version": "0.2.7",
   "description": "mirrorlist generator for Artix Linux",
   "keywords": [
     "artix",
@@ -9,7 +9,7 @@
   "license": "MIT",
   "repository": {
     "type": "git",
-    "url": "git+https://gitea.artixlinux.org/corysanin/artix-mlg.git"
+    "url": "https://gitea.artixlinux.org/corysanin/artix-mlg.git"
   },
   "author": {
     "name": "Cory Sanin",
@@ -18,6 +18,9 @@
   },
   "type": "module",
   "main": "./distribution/index.js",
+  "bin": {
+    "artix-mlg": "./bin/artix-mlg.mjs"
+  },
   "scripts": {
     "get-mirrorlist": "curl https://gitea.artixlinux.org/packages/artix-mirrorlist/raw/branch/master/mirrorlist > mirrorlist",
     "build": "npx tsc",
@@ -29,9 +32,7 @@
   ],
   "dependencies": {
     "country-code-lookup": "0.1.3",
-    "extract-tld": "1.1.2",
-    "ky": "1.8.2",
-    "tiny-spin": "1.0.2"
+    "extract-tld": "1.1.2"
   },
   "devDependencies": {
     "@types/node": "^24.1.0",

src/index.ts

@@ -1,8 +1,6 @@
 import path from 'path';
 import fsp from 'fs/promises';
-import ky from 'ky';
 import parseUrl from 'extract-tld';
-import { spin } from "tiny-spin";
 import { resolveCountry } from './resolveCountry.js';
 import MirrorsByRegion from './mirrorsByRegion.js';
 import type { PathLike } from 'fs';
@@ -11,16 +9,18 @@ import type { MirrorProfile, MirrorInput } from './mirrorProfile.ts';
 const inputFile = process.env['INPUT'] || path.join(process.cwd(), 'mirrors.json');
 const fixtureFile = process.env['FIXTURE'] || path.join(process.cwd(), 'mirrors.fixture.json');
 const mirrorList = process.env['MIRRORLIST'] || path.join(process.cwd(), 'mirrorlist');
+const mirrorMd = process.env['MIRRORMD'] || path.join(process.cwd(), 'mirrors.md');
+const mdHeadFile = process.env['MDHEADER'] || path.join(process.cwd(), 'head.md');
 const verbose = !!process.env['VERBOSE'];
-const skipIsoCheck = !!process.env['SKIPISO'];
 
 const protocolId: Record<Protocol, number> = {
     http: 1,
     rsync: 3,
-    https: 5
+    https: 5,
+    ftp: 9
 }
 
-type Protocol = 'http' | 'https' | 'rsync';
+type Protocol = 'http' | 'https' | 'rsync' | 'ftp';
 
 interface FixtureObject {
     pk: number;
@@ -64,35 +64,12 @@ interface UrlComponents {
     name: string;
     partial: string;
     protocol: Protocol;
+    mirrorListable: boolean;
 }
 
 type MirrorMap = { [name: string]: Mirror };
 
 let mirrorCounter = 0;
-let mirrorUrlCounter = 0;
-
-async function IsosPresent(url: string): Promise<boolean> {
-    if (skipIsoCheck) {
-        return false;
-    }
-    const pathTests = ['iso', 'isos', 'ISO', 'ISOs', 'ISOS'];
-    for (let i = 0; i < pathTests.length; i++) {
-        try {
-            await ky.get(`${url}${pathTests[i]}/`, { timeout: 5000 });
-            return true;
-        }
-        catch (err: unknown) {
-            if (!!err && typeof err === 'object' && !('response' in err)) {
-                console.error(`Failed to connect to ${url}`);
-                if (verbose) {
-                    console.error(err);
-                }
-                break;
-            }
-        }
-    }
-    return false;
-}
 
 function setMirrors(fixture: FixtureObject[]) {
     const mirrors: MirrorMap = {};
@@ -107,20 +84,23 @@ function setMirrors(fixture: FixtureObject[]) {
 }
 
 function processUrl(url: string): UrlComponents {
+    const mirrorListable: Protocol[] = ['http', 'https'];
+    const protocol = url.split(':')[0] as Protocol;
     return {
         name: parseUrl.parseUrl(url)?.domain,
         full: url,
-        protocol: url.split(':')[0] as Protocol,
-        partial: url.split('$repo')[0]
+        protocol,
+        partial: url.split('$repo')[0],
+        mirrorListable: mirrorListable.indexOf(protocol) >= 0
     }
 }
 
-function getMirror(url: UrlComponents): Mirror {
-    return mirrors[url.name] = {
-        pk: mirrors[url.name]?.pk || ++mirrorCounter,
+function getMirror(name: string): Mirror {
+    return mirrors[name] = {
+        pk: mirrors[name]?.pk || ++mirrorCounter,
         model: 'mirrors.Mirror',
         fields: {
-            name: url.name,
+            name: name,
             tier: -1,
             upstream: null,
             admin_email: '',
@@ -136,32 +116,32 @@ function getMirror(url: UrlComponents): Mirror {
     }
 }
 
-async function updateMirror(m: Mirror, profile: MirrorProfile, url: UrlComponents) {
-    m.fields.tier = profile.tier;
-    m.fields.admin_email = profile.admin_email || '';
-    m.fields.alternate_email = profile.alternate_email || '';
-    m.fields.notes = profile.notes || '';
+function updateMirror(m: Mirror, profile: MirrorProfile, url: UrlComponents) {
+    m.fields.tier = Math.max(m.fields.tier, profile.tier);
+    m.fields.admin_email = profile.admin_email || m.fields.admin_email;
+    m.fields.alternate_email = profile.alternate_email || m.fields.alternate_email;
+    m.fields.notes = profile.notes || m.fields.notes;
     m.fields.active ||= profile.active;
     m.fields.public &&= profile.public;
-    m.fields.isos ||= (skipIsoCheck && profile.isos) || (url.protocol !== 'rsync' && await IsosPresent(url.partial));
+    m.fields.isos ||= !!profile.stable_isos || !!profile.weekly_isos;
     if (url.protocol === 'rsync') {
         m.fields.rsync_user = profile.rsync_user || '';
         m.fields.rsync_password = profile.rsync_password || '';
     }
 }
 
-async function processMirrorProfile(m: MirrorProfile) {
+function processMirrorProfile(m: MirrorProfile, index: number) {
     const url: UrlComponents = processUrl(m.url);
-    const mirror: Mirror = getMirror(url);
-    await updateMirror(mirror, m, url);
+    const mirror: Mirror = getMirror(m.force_mirror_name || url.name);
+    updateMirror(mirror, m, url);
     const mirrorUrl: MirrorUrl = {
-        pk: mirrorUrlCounter++,
+        pk: index + 1,
         model: 'mirrors.MirrorUrl',
         fields: {
             url: url.partial,
             protocol: protocolId[url.protocol],
             mirror: mirror.pk,
-            country: resolveCountry(m.country)?.iso2 || null,
+            country: resolveCountry(m.country)?.iso2 || undefined,
             // populate ip fields with `mirrorresolv`
             has_ipv4: false,
             has_ipv6: false,
@@ -169,6 +149,18 @@ async function processMirrorProfile(m: MirrorProfile) {
         }
     };
     mirrorUrls.push(mirrorUrl);
+    pushMirrorProfile(m.force_mirror_name || url.name, m);
+}
+
+function pushMirrorProfile(name: string, m: MirrorProfile) {
+    const list: MirrorProfile[] = mirrorProfilesByMirrorName[name] ||= [];
+    list.push(m);
+}
+
+function updateUpstream(m: MirrorProfile) {
+    const url: UrlComponents = processUrl(m.url);
+    const mirror: Mirror = mirrors[m.force_mirror_name || url.name];
+    mirror.fields.upstream = (m.upstream && mirrors[m.upstream]?.pk) || mirror.fields.upstream;
 }
 
 function composeMirrorFixture(): FixtureObject[] {
@@ -186,12 +178,12 @@ function getDateTime(): string {
 }
 
 function generateMirrorlist(mirrors: MirrorProfile[] = []): string {
-    const httpMirrors = mirrors.filter(m => processUrl(m.url).protocol !== 'rsync');
-    const mirrorsByRegion = new MirrorsByRegion(mirrors);
+    const httpMirrors = mirrors.filter(m => processUrl(m.url).mirrorListable);
+    const mirrorsByRegion = new MirrorsByRegion(httpMirrors);
     const lines: string[] = [
         '##',
         '## Artix Linux repository mirrorlist',
-        `## Generated on ${getDateTime()}`,
+        `## Generated on ${getDateTime()} by artix-mlg`,
         '##',
         '',
         '# Artix mirrors',
@@ -203,7 +195,57 @@ function generateMirrorlist(mirrors: MirrorProfile[] = []): string {
     ];
     httpMirrors.filter(m => m.default).forEach(m => lines.push(`Server = ${m.url}`));
     lines.push('');
-    mirrorsByRegion.printMirrors(lines)
+    mirrorsByRegion.printMirrors(lines);
+    return lines.join('\n');
+}
+
+async function generateMirrorMd(): Promise<string> {
+    async function tryReadHeader(): Promise<string[]> {
+        try {
+            return [await fsp.readFile(mdHeadFile, 'utf-8')];
+        }
+        catch (err) {
+            if (verbose) {
+                console.error(err);
+            }
+            return [];
+        }
+    }
+    function pushTableRowIfTruthy(lines: string[], label: string, value: string | undefined | null | false) {
+        if (value) {
+            lines.push(`| ${label} | ${value} |`);
+        }
+    }
+    function findFirstWithChild<T, K extends keyof T>(profiles: T[], key: K): T[K] | undefined {
+        return profiles.find(p => !!p[key])?.[key];
+    }
+    const lines: string[] = await tryReadHeader();
+    lines.push('# Mirrors\n\nContact or other information for the mirrors of our repositories and ISOs.\n');
+    for (let mirrorName in mirrorProfilesByMirrorName) {
+        const profiles: MirrorProfile[] = mirrorProfilesByMirrorName[mirrorName];
+        const activeProfiles: MirrorProfile[] = profiles.filter(p => p.active);
+        const urls: string[] = profiles.map(p => {
+            const url = p.url.split('$repo')[0];
+            return p.active ? url : `${url} (inactive)`;
+        });
+        const upstream = findFirstWithChild(activeProfiles, 'upstream');
+        lines.push(`### ${mirrorName}`);
+        lines.push(`| Mirror | ${mirrorName} |`);
+        lines.push('| ------ | ------------- |');
+        if (upstream) {
+            lines.push(`| Sync Source | [${upstream}](#${upstream.replaceAll('.', '')}) |`);
+        }
+        lines.push(`| URLs | ${urls.join('<br>')} |`);
+        pushTableRowIfTruthy(lines, 'Provides Stable ISO', findFirstWithChild(activeProfiles, 'stable_isos'));
+        pushTableRowIfTruthy(lines, 'Provides Weekly ISO', findFirstWithChild(activeProfiles, 'weekly_isos'));
+        // pushTableRowIfTruthy(lines, 'Bandwidth', findFirstWithChild(activeProfiles, 'bandwidth'));
+        // pushTableRowIfTruthy(lines, 'Frequency', findFirstWithChild(activeProfiles, 'frequency'));
+        // pushTableRowIfTruthy(lines, 'Hosted by', findFirstWithChild(activeProfiles, 'org'));
+        pushTableRowIfTruthy(lines, 'Location', findFirstWithChild(activeProfiles, 'country') || findFirstWithChild(profiles, 'country'));
+        pushTableRowIfTruthy(lines, 'Contact Details', findFirstWithChild(activeProfiles, 'admin_email') || findFirstWithChild(profiles, 'admin_email'));
+        pushTableRowIfTruthy(lines, 'Altenate Contact Details', findFirstWithChild(activeProfiles, 'alternate_email') || findFirstWithChild(profiles, 'alternate_email'));
+        lines.push('');
+    }
     return lines.join('\n');
 }
 
@@ -223,18 +265,17 @@ async function tryReadFileOrDefault<T>(f: PathLike, d: T): Promise<T> {
 const fixture: FixtureObject[] = await tryReadFileOrDefault<FixtureObject[]>(fixtureFile, []);
 const mirrors: MirrorMap = setMirrors(fixture);
 const mirrorUrls: MirrorUrl[] = [];
+const mirrorProfilesByMirrorName: { [mirrorName: string]: MirrorProfile[] } = {};
 
 async function main() {
     const input: MirrorInput = JSON.parse(await fsp.readFile(inputFile, { encoding: 'utf-8' }));
 
-    const stopSpin = spin("Checking for ISOs...");
-    for (let i = 0; i < input.mirrors?.length; i++) {
-        await processMirrorProfile(input.mirrors[i]);
-    }
-    stopSpin();
+    input.mirrors.forEach(processMirrorProfile);
+    input.mirrors.forEach(updateUpstream);
 
     await fsp.writeFile(fixtureFile, JSON.stringify(composeMirrorFixture(), null, 4));
-    await fsp.writeFile(mirrorList, generateMirrorlist(input.mirrors?.filter(m => m.public && m.active) || []));
+    await fsp.writeFile(mirrorList, generateMirrorlist(input.mirrors?.filter(m => m.public && m.active && !m.suppress) || []));
+    await fsp.writeFile(mirrorMd, await generateMirrorMd());
 }
 
 export default main;

src/mirrorProfile.ts

@@ -2,15 +2,19 @@ interface MirrorProfile {
     url: string;
     tier: number;
     country: string;
+    upstream: string;
     public: boolean;
     active: boolean;
     default: boolean;
+    suppress?: boolean;
     admin_email?: string;
     alternate_email?: string;
-    isos?: boolean;
+    stable_isos?: string;
+    weekly_isos?: string;
     rsync_user?: string;
     rsync_password?: string;
     notes?: string;
+    force_mirror_name?: string;
 }
 
 interface MirrorInput {