diff --git a/Makefile b/Makefile index 0447644..9486708 100644 --- a/Makefile +++ b/Makefile @@ -45,6 +45,10 @@ install: deno install --unstable -A -f -n ats ./cli/ats.js deno install --unstable -A -f -n ats-repo ./cli/ats-repo.js +did-snapshot: + deno run -A ./backend/did-snapshot.js + tree /corn/atscan-db/snapshots/public/did + typesense-init: deno run --unstable --allow-net --allow-read ./backend/typesense-init.js diff --git a/backend/did-snapshot-publish.sh b/backend/did-snapshot-publish.sh new file mode 100755 index 0000000..532d44d --- /dev/null +++ b/backend/did-snapshot-publish.sh @@ -0,0 +1,9 @@ +#!/bin/bash +cd $1 +echo $4 +mkdir -p $4 +#tar --zstd -cf $3.tar.zst $2 +#sha256sum $3.tar.zst > $3.tar.zst.sha256 +tar --gzip -cf $3.tar.gz $2 +sha256sum $3.tar.gz > $3.tar.gz.sha256 +mv $3* $4 \ No newline at end of file diff --git a/backend/did-snapshot.js b/backend/did-snapshot.js new file mode 100644 index 0000000..600078e --- /dev/null +++ b/backend/did-snapshot.js @@ -0,0 +1,73 @@ +import { ATScan } from "./lib/atscan.js"; +import { ensureDir } from "https://deno.land/std@0.192.0/fs/ensure_dir.ts"; +import { emptyDir } from "https://deno.land/std@0.192.0/fs/empty_dir.ts"; + + +const ats = new ATScan(); +ats.debug = true; +await ats.init(); + +const limit = 1000; +let total = await ats.db.did.count() +let i = 0 + +const SNAPSHOT_DIR = "/corn/atscan-db/snapshots" +const SNAPSHOT_PREPARE_DIR = SNAPSHOT_DIR + "/did/prepare" +const SNAPSHOT_PUBLISH_DIR = SNAPSHOT_DIR + "/public/did" + +await emptyDir(SNAPSHOT_PREPARE_DIR) + +async function dump (root) { + async function iterator () { + for (const x of await ats.db.did.find().limit(limit).skip(i).toArray()) { + const src = x.src.replace(/^https?:\/\//, '') + const shards = x.did.substring(8) + const dir = [root, src, ...[shards.substring(0,2), shards.substring(2,2)]].join("/") + await ensureDir(dir) + await Deno.writeTextFile(`${dir}/${x.did.replace(/^did:plc:/,'')}.json`, JSON.stringify(x, null, 2)) + i++; + } + } + + let done = false + while (!done) { + console.log(i + ' ...') + await iterator() + + if (i >= total) { + done = true + } + } +} + +async function _(c, opts) { + console.log(`running ${c} ${opts.args.join(" ")}`); + const cmd = new Deno.Command(c, opts); + const { stderr, stdout } = await cmd.output() + + const err = new TextDecoder().decode(stderr) + console.log(err.trim()) + + const out = new TextDecoder().decode(stdout) + console.log(out.trim()) + + return out +} + +async function compress (dir) { + for await (const f of Deno.readDir(dir)) { + const fb = (new Date).toISOString().substring(0, 10) + '_' + f.name + const pd = SNAPSHOT_PUBLISH_DIR + '/' + f.name + if (!f.isDirectory) { + continue + } + await _("bash", {args: [`/corn/atscan/backend/did-snapshot-publish.sh`, dir, f.name, fb, pd]}) + } + +} + +await dump(SNAPSHOT_PREPARE_DIR) +await compress(SNAPSHOT_PREPARE_DIR) + +console.log('done') +Deno.exit() \ No newline at end of file diff --git a/backend/plc-crawler.js b/backend/plc-crawler.js index 22142da..9d92fef 100644 --- a/backend/plc-crawler.js +++ b/backend/plc-crawler.js @@ -99,7 +99,7 @@ async function processPlcExport(ats, plc, after = null) { const didId = [data.did, didRev].join("@"); if (pdsFound) { if (!pdsFound.plcs.includes(plc.url)) { - pdsFound.plcs.push(plcUrl); + pdsFound.plcs.push(plc.url); console.log( `${ (new Date()).toISOString()