did snapshots

Makefile

@@ -45,6 +45,10 @@ install:
 	deno install --unstable -A -f -n ats ./cli/ats.js
 	deno install --unstable -A -f -n ats-repo ./cli/ats-repo.js
 
+did-snapshot:
+	deno run -A ./backend/did-snapshot.js
+	tree /corn/atscan-db/snapshots/public/did
+
 typesense-init:
 	deno run --unstable --allow-net --allow-read ./backend/typesense-init.js
 

backend/did-snapshot-publish.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+cd $1
+echo $4
+mkdir -p $4
+#tar --zstd -cf $3.tar.zst $2
+#sha256sum $3.tar.zst > $3.tar.zst.sha256
+tar --gzip -cf $3.tar.gz $2
+sha256sum $3.tar.gz > $3.tar.gz.sha256
+mv $3* $4

backend/did-snapshot.js

@@ -0,0 +1,73 @@
+import { ATScan } from "./lib/atscan.js";
+import { ensureDir } from "https://deno.land/std@0.192.0/fs/ensure_dir.ts";
+import { emptyDir } from "https://deno.land/std@0.192.0/fs/empty_dir.ts";
+
+
+const ats = new ATScan();
+ats.debug = true;
+await ats.init();
+
+const limit = 1000;
+let total = await ats.db.did.count()
+let i = 0
+
+const SNAPSHOT_DIR = "/corn/atscan-db/snapshots"
+const SNAPSHOT_PREPARE_DIR = SNAPSHOT_DIR + "/did/prepare"
+const SNAPSHOT_PUBLISH_DIR = SNAPSHOT_DIR + "/public/did"
+
+await emptyDir(SNAPSHOT_PREPARE_DIR)
+
+async function dump (root) {
+    async function iterator () {
+        for (const x of await ats.db.did.find().limit(limit).skip(i).toArray()) {
+            const src = x.src.replace(/^https?:\/\//, '')
+            const shards = x.did.substring(8)
+            const dir = [root, src, ...[shards.substring(0,2), shards.substring(2,2)]].join("/")
+            await ensureDir(dir)
+            await Deno.writeTextFile(`${dir}/${x.did.replace(/^did:plc:/,'')}.json`, JSON.stringify(x, null, 2))
+            i++;
+        }
+    }
+
+    let done = false
+    while (!done) {
+        console.log(i + ' ...')
+        await iterator()
+
+        if (i >= total) {
+            done = true
+        }
+    }
+}
+
+async function _(c, opts) {
+    console.log(`running ${c} ${opts.args.join(" ")}`);
+    const cmd = new Deno.Command(c, opts);
+    const { stderr, stdout } = await cmd.output()
+
+    const err = new TextDecoder().decode(stderr)
+    console.log(err.trim())
+    
+    const out = new TextDecoder().decode(stdout) 
+    console.log(out.trim())
+
+    return out
+}
+
+async function compress (dir) {
+    for await (const f of Deno.readDir(dir)) {
+        const fb = (new Date).toISOString().substring(0, 10) + '_' + f.name
+        const pd = SNAPSHOT_PUBLISH_DIR + '/' + f.name
+        if (!f.isDirectory) {
+            continue
+        }
+        await _("bash", {args: [`/corn/atscan/backend/did-snapshot-publish.sh`, dir, f.name, fb, pd]})
+      }
+
+}
+
+await dump(SNAPSHOT_PREPARE_DIR)
+await compress(SNAPSHOT_PREPARE_DIR)
+
+console.log('done')
+Deno.exit()

backend/plc-crawler.js

@@ -99,7 +99,7 @@ async function processPlcExport(ats, plc, after = null) {
       const didId = [data.did, didRev].join("@");
       if (pdsFound) {
         if (!pdsFound.plcs.includes(plc.url)) {
-          pdsFound.plcs.push(plcUrl);
+          pdsFound.plcs.push(plc.url);
           console.log(
             `${
               (new Date()).toISOString()