texas node

.env.example

@@ -1,4 +1,9 @@
 IPINFO_TOKEN=YOUR_TOKEN
 MONGODB_URL=mongodb://127.0.0.1:27017
+NATS_SERVERS=XXXX
 BLUESKY_USERNAME=YOUR_USERNAME
-BLUESKY_PASSWORD=YOUR_PASSWORD
+BLUESKY_PASSWORD=YOUR_PASSWORD
+INFLUXDB_HOST=http://localhost:8086
+INFLUXDB_TOKEN=XXXX
+INFLUXDB_ORG=XXXX
+INFLUXDB_BUCKET=XXXX

backend/api.js

@@ -80,7 +80,7 @@ router
         |> range(start: -1d)
         |> filter(fn: (r) => r["_measurement"] == "pds_response_time")
         |> filter(fn: (r) => r["pds"] == "${item.host}")
-        |> aggregateWindow(every: 1m, fn: mean, createEmpty: false)`;
+        |> aggregateWindow(every: 3m, fn: mean, createEmpty: true)`;
 
     item.responseTimesDay = await ats.influxQuery.collectRows(query);
 
@@ -95,13 +95,14 @@ router
       did: { key: "did" },
       lastMod: { key: "lastMod" },
       pds: { key: "pds" },
+      size: { key: "repo.size" },
     };
 
     let inputSort = ctx.request.url.searchParams.get("sort");
     let inputSortDirection = 1;
     if (inputSort && inputSort.startsWith("!")) {
       inputSortDirection = -1;
-      inputSort.replace(/^!/, "");
+      inputSort = inputSort.replace(/^!/, "");
     }
     let inputSortConfig = null;
     if (inputSort) {
@@ -155,7 +156,7 @@ router
     }
 
     //console.log(JSON.stringify(query, null, 2), { sort, limit });
-    console.log(JSON.stringify(query));
+    //console.log(JSON.stringify({ query, sort, inputSort, inputSortConfig }));
     const count = await ats.db.did.count(query);
 
     for (

backend/lib/atscan.js

@@ -140,7 +140,7 @@ export class ATScan {
   }
 
   async writeInflux (name, type, value, tags = []) {
-    const point = `${name},${tags.map(t => t.join('=')).join(', ')} value=${value} ${Date.now()}`;
+    const point = `${name},${tags.map(t => t.join('=')).join(',')} value=${value} ${Date.now()}`;
     const resp = await fetch(`${Deno.env.get('INFLUXDB_HOST')}/api/v2/write?org=${Deno.env.get('INFLUXDB_ORG')}&bucket=${Deno.env.get('INFLUXDB_BUCKET')}&precision=ms`, {
       method: 'POST',
       headers: {

backend/pds-crawler.js

@@ -1,18 +1,93 @@
 import { ATScan } from "./lib/atscan.js";
 import { pooledMap } from "https://deno.land/std/async/mod.ts";
 import { timeout } from "./lib/utils.js";
+import {
+  connect,
+  JSONCodec,
+  StringCodec,
+} from "https://deno.land/x/nats/src/mod.ts";
 import "https://deno.land/std@0.192.0/dotenv/load.ts";
 
 const WAIT = 1000 * 60 * 2;
 const TIMEOUT = 5000;
 
+const nc = await connect({
+  servers: Deno.env.get("NATS_SERVERS"),
+});
+const jc = JSONCodec();
+console.log(`connected to ${nc.getServer()}`);
+
+const hosts = {
+  local: {},
+  texas: {},
+};
+
+async function crawlUrl(url, host = "local") {
+  if (host === "local") {
+    try {
+      const [, ms] = await timeout(
+        TIMEOUT,
+        fetch(url, {
+          method: "OPTIONS",
+          headers: {
+            "User-Agent": "ATScan Crawler",
+            "connection": "keep-alive",
+            keepalive: "timeout=5, max=1000",
+          },
+        }),
+      );
+    } catch (e) {
+      return { err: "timeout" };
+    }
+    let res, data, ms, err;
+    try {
+      [res, ms] = await timeout(
+        TIMEOUT,
+        fetch(url, {
+          headers: {
+            "User-Agent": "ATScan Crawler",
+          },
+        }),
+      );
+      if (res) {
+        data = await res.json();
+      }
+    } catch (e) {
+      err = e.message;
+    }
+    return {
+      err,
+      data,
+      ms,
+    };
+  }
+  const hostConfig = hosts[host];
+  if (!hostConfig) {
+    console.error(`Unknown host: ${host}`);
+    return { err: "unknown host" };
+  }
+  const resp = await nc.request(`ats-nodes.${host}.http`, jc.encode({ url }), {
+    timeout: 60000,
+  });
+  const { err, data, ms } = jc.decode(resp.data);
+  return { err, data, ms };
+}
+
 async function crawl(ats) {
   const arr = await ats.db.pds.find().toArray();
   const results = pooledMap(25, arr.slice(0, 1000), async (i) => {
     let err = null;
-    let res, data, ms;
+
+    if (i.url.match(/^https?:\/\/(localhost|example.com)/)) {
+      err = "not allowed domain";
+      await ats.db.pds.updateOne({ url: i.url }, {
+        $set: { "inspect.current": { err } },
+      });
+      return;
+    }
 
     const host = i.url.replace(/^https?:\/\//, "");
+
     if (!i.dns) {
       console.log("sending dns request: ", i.url);
       let dns =
@@ -44,54 +119,41 @@ async function crawl(ats) {
       }
     }
 
-    if (i.url.match(/^https?:\/\/(localhost|example.com)/)) {
-      err = "not allowed domain";
-    }
     if (!i.dns.Answer) {
       err = "not existing domain";
     }
 
-    if (!err) {
-      const url = `${i.url}/xrpc/com.atproto.server.describeServer`;
-      try {
-        [res, ms] = await timeout(
-          TIMEOUT,
-          fetch(url, {
-            headers: {
-              "User-Agent": "ATScan Crawler",
-            },
-          }),
-        );
-        if (res) {
-          data = await res.json();
+    const url = `${i.url}/xrpc/com.atproto.server.describeServer`;
+    await Promise.all(
+      Object.keys(hosts).map(async (chost) => {
+        const { err, data, ms } = await crawlUrl(url, chost);
+        const inspect = {
+          err,
+          data,
+          ms,
+          time: new Date().toISOString(),
+        };
+        if (chost === "local") {
+          const dbSet = { "inspect.current": inspect };
+          if (!err && data) {
+            dbSet["inspect.lastOnline"] = (new Date()).toISOString();
+          }
+          await ats.db.pds.updateOne({ url: i.url }, {
+            $set: dbSet,
+          });
         }
-      } catch (e) {
-        err = e.message;
-      }
-    }
-    const inspect = {
-      err,
-      data,
-      ms,
-      time: new Date().toISOString(),
-    };
-    const dbSet = { "inspect.current": inspect };
-    if (!err && data) {
-      dbSet["inspect.lastOnline"] = (new Date()).toISOString();
-    }
-    await ats.db.pds.updateOne({ url: i.url }, {
-      $set: dbSet,
-    });
-    if (ms && Number(ms) > 0) {
-      await ats.writeInflux("pds_response_time", "intField", Number(ms), [[
-        "pds",
-        host,
-      ]]);
-    }
-    console.log(
-      `-> ${i.url} ${ms ? "[" + ms + "ms]" : ""} ${
-        err ? "error = " + err : ""
-      }`,
+        if (ms && Number(ms) > 0) {
+          await ats.writeInflux("pds_response_time", "intField", Number(ms), [
+            ["pds", host],
+            ["crawler", chost],
+          ]);
+        }
+        console.log(
+          `[${chost}] -> ${i.url} ${ms ? "[" + ms + "ms]" : ""} ${
+            err ? "error = " + err : ""
+          }`,
+        );
+      }),
     );
   });
   for await (const _ of results) {}

frontend/src/lib/components/DIDTable.svelte

@@ -7,7 +7,8 @@
 		identicon,
 		formatNumber,
 		customTableMapper,
-		getDIDProfileUrl
+		getDIDProfileUrl,
+		filesize
 	} from '$lib/utils.js';
 	export let sourceData;
 	export let data;
@@ -80,12 +81,28 @@
 		if (key === 'url') {
 			val = `/${row.did}`;
 		}
+		if (key === 'size') {
+			val =
+				'<div class="text-lg">' +
+				(row.repo?.size
+					? filesize(row.repo.size) +
+					  '</div><div>' +
+					  formatNumber(
+							Object.keys(row.repo.collections).reduce((t, c) => (t += row.repo.collections[c]), 0)
+					  ) +
+					  ' items</div>'
+					: '-');
+		}
 		return val;
 	}
 	$: tableSimple = {
 		// A list of heading labels.
-		head: ['', ['DID', 'did'], ['PDS (PLC)', 'pds'], ['Updated', 'lastMod']],
-		body: customTableMapper(sourceData || [], ['img', 'did', 'srcHost', 'lastMod'], tableMap),
+		head: ['', ['DID', 'did'], ['PDS (PLC)', 'pds'], ['Repo size', 'size'], ['Updated', 'lastMod']],
+		body: customTableMapper(
+			sourceData || [],
+			['img', 'did', 'srcHost', 'size', 'lastMod'],
+			tableMap
+		),
 		meta: customTableMapper(sourceData || [], ['did_raw', 'url'], tableMap)
 	};
 </script>

frontend/src/routes/did:[id]/+page.svelte

@@ -1,5 +1,5 @@
 <script>
-	import { dateDistance, identicon, getDIDProfileUrl, filesize } from '$lib/utils.js';
+	import { dateDistance, identicon, getDIDProfileUrl, filesize, formatNumber } from '$lib/utils.js';
 	import { Table } from '@skeletonlabs/skeleton';
 	import { tableMapperValues, tableSourceValues } from '@skeletonlabs/skeleton';
 	import SourceSection from '$lib/components/SourceSection.svelte';
@@ -106,17 +106,28 @@
 					</tr>
 					<tr>
 						<th class="text-right">Commits</th>
-						<td>{item.repo.commits}</td>
+						<td>{formatNumber(item.repo.commits)}</td>
 					</tr>
 					<tr>
 						<th class="text-right">Size</th>
 						<td>{filesize(item.repo?.size)}</td>
 					</tr>
+					<tr>
+						<th class="text-right">Records</th>
+						<td
+							>{formatNumber(
+								Object.keys(item.repo?.collections).reduce(
+									(t, c) => (t += item.repo.collections[c]),
+									0
+								)
+							)} items</td
+						>
+					</tr>
 					<tr>
 						<th class="text-right">Collections</th>
 						<td
 							>{Object.keys(item.repo?.collections)
-								.map((c) => `${item.repo.collections[c]} ${c}`)
+								.map((c) => `${formatNumber(item.repo.collections[c])} ${c}`)
 								.join(', ')}</td
 						>
 					</tr>
@@ -131,5 +142,5 @@
 		<div>No repository info yet.</div>
 	{/if}
 
-	<SourceSection {data} model="did" />
+	<SourceSection {data} model="did" hide="true" />
 </BasicPage>

frontend/src/routes/dids/+page.js

@@ -13,22 +13,18 @@ export async function load({ fetch, url, parent }) {
 	if (sort) {
 		args.push(`sort=${sort}`);
 	}
-
 	const res = await fetch(`${config.api}/dids` + (args.length > 0 ? '?' + args.join('&') : ''), {
 		headers: { 'x-ats-wrapped': 'true' }
 	});
 	const json = await res.json();
-	const totalCount = json.count;
-	const did = _.orderBy(json.items, ['time'], ['desc']);
 	let onlySandbox = false;
 	if (q?.match(/fed:sandbox/)) {
 		onlySandbox = true;
 		q = q.replace('fed:sandbox', '').trim();
 	}
-
 	return {
-		did,
-		totalCount,
+		did: json.items,
+		totalCount: json.count,
 		q,
 		sort,
 		onlySandbox

frontend/src/routes/pds/[host]/+page.svelte

@@ -97,17 +97,26 @@
 		});
 	}
 
+	const crawlers = {
+		local: {
+			location: 'Central Europe (Prague, CZ)'
+		},
+		texas: {
+			location: 'North America (Texas, US)'
+		}
+	};
+
 	$: chartResponseTimes = {
 		animationDuration: 500,
 		title: {
 			text: `${item.host} response times in last 24 hours`
 		},
 		tooltip: {
-			trigger: 'axis',
-			formatter: '{b}: {c} ms'
+			trigger: 'axis'
+			//formatter: '{b}: {c} ms'
 		},
 		legend: {
-			data: [chartHost]
+			data: Object.keys(crawlers).map((c) => crawlers[c].location)
 		},
 		grid: {
 			left: '2%',
@@ -123,7 +132,7 @@
 		xAxis: {
 			type: 'category',
 			boundaryGap: false,
-			data: item.responseTimesDay?.map((r) => r._time) || []
+			data: item.responseTimesDay?.filter((r) => r.table === 0).map((r) => r._time) || []
 		},
 		yAxis: {
 			type: 'value',
@@ -131,16 +140,16 @@
 				formatter: '{value} ms'
 			}
 		},
-		series: [
-			{
-				name: chartHost,
+		series: Object.keys(crawlers).map((crawler) => {
+			const crawlerOptions = crawlers[crawler];
+			return {
+				name: crawlerOptions.location,
 				type: 'line',
-				stack: 'ms',
-				data: item.responseTimesDay?.map((r) => r._value) || []
-			}
-		]
+				//stack: 'ms',
+				data: item.responseTimesDay?.filter((r) => r.crawler === crawler).map((r) => r._value) || []
+			};
+		})
 	};
-	const chartHost = 'Central Europe (Prague, CZ)';
 </script>
 
 <BasicPage {data} title={item.host} {breadcrumb} noHeader="true">