pds size column, update indexer & car

backend/indexer.js

@@ -1,15 +1,30 @@
 import { ATScan } from "./lib/atscan.js";
 import whoiser from "npm:whoiser";
 
-const wait = 60 * 15;
+const wait = 60 * 5;
 
 async function index(ats) {
   for (const pds of await ats.db.pds.find().toArray()) {
     const didsCount = await ats.db.did.countDocuments({
       "pds": { $in: [pds.url] },
     });
-    //console.log(`${pds.url}: ${didsCount}`);
-    await ats.db.pds.updateOne({ url: pds.url }, { $set: { didsCount } });
+
+    const stages = [
+      { $match: { pds: { $in: [pds.url] } } },
+      {
+        $group: {
+          _id: "$groupField",
+          sum: {
+            $sum: "$repo.size",
+          },
+        },
+      },
+    ];
+    const sizeRes = await ats.db.did.aggregate(stages).toArray();
+    const size = sizeRes[0].sum;
+    //console.log(`${pds.url}: ${size}`);
+
+    await ats.db.pds.updateOne({ url: pds.url }, { $set: { didsCount, size } });
   }
   console.log("indexer round finished");
   //console.log(await whoiser("dev.otaso-sky.blue"));
@@ -33,13 +48,5 @@ if (Deno.args[0] === "daemon") {
   await ats.init();
   await index(ats);
 
-  for (
-    const did of await ats.db.did.find({ lastMod: { $exists: false } })
-      .toArray()
-  ) {
-    await ats.db.did.updateOne({ _id: did._id }, {
-      $set: { lastMod: did.revs[did.revs.length - 1].createdAt },
-    });
-    console.log(`${did} updated ${did.revs[did.revs.length - 1].createdAt}`);
-  }
+  Deno.exit(0);
 }

backend/lib/car.js

@@ -15,6 +15,8 @@ const colsDefs = {
   "app.bsky.graph.follow": { key: "follow" },
   "app.bsky.feed.generator": { key: "generator" },
   "app.bsky.graph.block": { key: "block" },
+  "app.bsky.graph.list": { key: "list" },
+  "app.bsky.graph.listitem": { key: "listitem" },
 };
 
 export async function readRaw(data) {

backend/repo-crawler.js

@@ -21,7 +21,7 @@ async function crawl(ats) {
     $or: [{ "repo.time": { $lte: expiry } }, { "repo": { $exists: false } }],
   }).limit(10000).toArray();
 
-  const results = pooledMap(8, _.shuffle(dids), async (didInfo) => {
+  const results = pooledMap(1, _.shuffle(dids), async (didInfo) => {
     const did = didInfo.did;
     const signingKey = didInfo.revs[didInfo.revs.length - 1].operation
       .verificationMethods?.atproto;
@@ -37,6 +37,7 @@ async function crawl(ats) {
 
     // fetch remote repo
     const url = `${pds}/xrpc/com.atproto.sync.getRepo?did=${did}`;
+    console.log(url);
     let repoRes;
     try {
       [repoRes] = await timeout(20 * 1000, fetch(url));
@@ -49,6 +50,7 @@ async function crawl(ats) {
       });
       return;
     }
+    console.log(`downloaded: ${url}`);
     if (!repoRes.ok) {
       let message = null;
       if ([403, 500].includes(repoRes.status)) {
@@ -70,6 +72,7 @@ async function crawl(ats) {
     try {
       repo = await inspect(data, did, signingKey);
     } catch (e) {
+      console.error(e);
       await ats.db.did.updateOne({ did }, {
         $set: { repo: { error: e.message, time: new Date() } },
       });

frontend/src/lib/components/PDSTable.svelte

@@ -7,7 +7,8 @@
 		identicon,
 		formatNumber,
 		customTableMapper,
-		getPDSStatus
+		getPDSStatus,
+		filesize
 	} from '$lib/utils.js';
 	export let sourceData;
 	export let data;
@@ -70,6 +71,9 @@
 		if (key === 'didsCount') {
 			val = `<a href="/dids?q=pds:${row.host}" class="anchor">${formatNumber(val)}</a>`;
 		}
+		if (key === 'size') {
+			val = row.size ? filesize(row.size) : '-';
+		}
 		if (key === 'lastOnline' && row.inspect) {
 			val = `<span class="text-xs">${
 				row.inspect.lastOnline && !row.inspect?.current?.err
@@ -100,6 +104,7 @@
 			'',
 			['Host', 'host'],
 			['DIDs', 'didsCount'],
+			['Size', 'size'],
 			['Location', 'country'],
 			['PLCs (User Domains)', 'plcs'],
 			['Resp. time', 'responseTime'],
@@ -107,7 +112,17 @@
 		],
 		body: customTableMapper(
 			sourceData,
-			['fed', 'status', 'host', 'didsCount', 'location', 'plcs', 'responseTime', 'lastOnline'],
+			[
+				'fed',
+				'status',
+				'host',
+				'didsCount',
+				'size',
+				'location',
+				'plcs',
+				'responseTime',
+				'lastOnline'
+			],
 			tableMap
 		),
 		meta: customTableMapper(sourceData, ['host_raw', 'url'], tableMap)

frontend/src/routes/pds/+page.svelte

@@ -79,7 +79,6 @@
 			}
 			base = orderBy(base, [sortKey], [sortDirection === -1 ? 'desc' : 'asc']);
 		} else {
-			console.log('x');
 			base = orderBy(base, ['env', 'err', 'didsCount'], ['asc', 'asc', 'desc']);
 		}