This commit is contained in:
tree 2023-07-12 17:22:49 +00:00
rodič ba593bd9f7
revize 89e1fd7e0d
16 změnil soubory, kde provedl 564 přidání a 31 odebrání

Zobrazit soubor

@ -23,7 +23,7 @@ if (Number(ats.env.PORT) === 6677) {
if (!item) {
continue;
}
Object.assign(item, prepareObject("did", item));
Object.assign(item, await prepareObject("did", item));
item.revs = [item.revs[item.revs.length - 1]];
ats.nats.publish(
`ats.api.did.${
@ -45,7 +45,7 @@ if (Number(ats.env.PORT) === 6677) {
if (!item) {
continue;
}
Object.assign(item, prepareObject("pds", item));
Object.assign(item, await prepareObject("pds", item));
ats.nats.publish(
`ats.api.pds.${sub === "ats.service.pds.create" ? "create" : "update"}`,
codec.encode(item),
@ -112,7 +112,7 @@ function getPDSStatus(item) {
return offlineNum > 0 ? "degraded" : "online";
}
function prepareObject(type, item) {
async function prepareObject(type, item) {
switch (type) {
case "pds":
item.host = item.url.replace(/^https?:\/\//, "");
@ -141,6 +141,21 @@ function prepareObject(type, item) {
item.srcHost = item.src.replace(/^https?:\/\//, "");
item.fed = findDIDFed(item);
break;
case "bgs":
item.host = item.url.replace(/^https?:\/\//, "");
item.status = await ats.redis.hGetAll(`ats:bgs:${item.host}`);
break;
case "plc":
item.host = item.url.replace(/^https?:\/\//, "");
item.didsCount = await ats.db.did.countDocuments({ src: item.url });
item.pdsCount = await ats.db.pds.countDocuments({
plcs: { $in: [item.url] },
});
item.lastUpdate =
(await ats.db.meta.findOne({ key: `lastUpdate:${item.url}` })).value;
break;
}
return item;
}
@ -160,7 +175,7 @@ router
console.error("PDS without url? ", item);
continue;
}
Object.assign(item, prepareObject("pds", item));
Object.assign(item, await prepareObject("pds", item));
//item.didsCount = await ats.db.did.countDocuments({ 'pds': { $in: [ item.url ] }})
out.push(item);
}
@ -173,7 +188,7 @@ router
if (!item) {
return ctx.response.code = 404;
}
Object.assign(item, prepareObject("pds", item));
Object.assign(item, await prepareObject("pds", item));
ctx.response.body = item;
perf(ctx);
@ -309,7 +324,7 @@ router
const did
of (await ats.db.did.find(query).sort(sort).limit(limit).toArray())
) {
Object.assign(did, prepareObject("did", did));
Object.assign(did, await prepareObject("did", did));
out.push(did);
}
@ -330,6 +345,98 @@ router
: out;
perf(ctx);
})
.get("/bgs", async (ctx) => {
const items = await Promise.all(
ats.ecosystem.data["bgs-instances"].map(async (item) => {
return Object.assign(item, await prepareObject("bgs", item));
}),
);
ctx.response.body = items;
perf(ctx);
})
.get("/bgs/:host", async (ctx) => {
const item = ats.ecosystem.data["bgs-instances"].find((f) =>
f.url === "https://" + ctx.params.host
);
if (!item) {
return ctx.response.code = 404;
}
Object.assign(item, await prepareObject("bgs", item));
ctx.response.body = item;
perf(ctx);
})
.get("/bgs/:host/ops", async (ctx) => {
const item = ats.ecosystem.data["bgs-instances"].find((f) =>
f.url === "https://" + ctx.params.host
);
if (!item) {
return ctx.response.code = 404;
}
const host = item.url.replace(/^https?:\/\//, "");
const allowedRanges = ["1h", "24h", "7d", "30d"];
const rangesAggregation = ["15s", "5m", "30m", "2h"];
const userRange = ctx.request.url.searchParams.get("range");
const range = userRange && allowedRanges.includes(userRange)
? userRange
: "24h";
const aggregation = rangesAggregation[allowedRanges.indexOf(range)];
let query = `
from(bucket: "ats-nodes")
|> range(start: -${range})
|> filter(fn: (r) => r["_measurement"] == "firehose_event")
|> filter(fn: (r) => r["server"] == "snowden")
|> filter(fn: (r) => r["bgs"] == "${host}")
|> filter(fn: (r) => r["_field"] == "value")
|> derivative(unit: 1s, nonNegative: true)
|> aggregateWindow(every: ${aggregation}, fn: mean, createEmpty: true)`;
if (!ctx.request.url.searchParams.get("complex")) {
query += `
|> group(columns: ["_time"], mode:"by")
|> sum()
|> group()`;
}
const data = await ats.influxQuery.collectRows(query);
ctx.response.body = { range, aggregation, data };
perf(ctx);
})
.get("/bgs/:host/postLatency", async (ctx) => {
const item = ats.ecosystem.data["bgs-instances"].find((f) =>
f.url === "https://" + ctx.params.host
);
if (!item) {
return ctx.response.code = 404;
}
const host = item.url.replace(/^https?:\/\//, "");
const allowedRanges = ["1h", "24h", "7d", "30d"];
const rangesAggregation = ["15s", "5m", "30m", "2h"];
const userRange = ctx.request.url.searchParams.get("range");
const range = userRange && allowedRanges.includes(userRange)
? userRange
: "24h";
const aggregation = rangesAggregation[allowedRanges.indexOf(range)];
let query = `
from(bucket: "ats-nodes")
|> range(start: -${range})
|> filter(fn: (r) => r["_measurement"] == "post_latency")
|> filter(fn: (r) => r["server"] == "snowden")
|> filter(fn: (r) => r["bgs"] == "${host}")
|> filter(fn: (r) => r["_field"] == "value")
|> aggregateWindow(every: ${aggregation}, fn: mean, createEmpty: true)`;
if (!ctx.request.url.searchParams.get("complex")) {
query += `
|> group(columns: ["_time"], mode:"by")
|> sum()
|> group()`;
}
const data = await ats.influxQuery.collectRows(query);
ctx.response.body = { range, aggregation, data };
perf(ctx);
})
.get("/feds", (ctx) => {
ctx.response.body = ats.ecosystem.data.federations;
perf(ctx);
@ -356,21 +463,25 @@ router
ctx.response.body = item;
perf(ctx);
})
.get("/plc", async (ctx) => {
.get("/plcs", async (ctx) => {
const out = [];
for (const plc of ats.ecosystem.data["plc-directories"]) {
plc.host = plc.url.replace(/^https?:\/\//, "");
plc.didsCount = await ats.db.did.countDocuments({ src: plc.url });
plc.pdsCount = await ats.db.pds.countDocuments({
plcs: { $in: [plc.url] },
});
plc.lastUpdate =
(await ats.db.meta.findOne({ key: `lastUpdate:${plc.url}` })).value;
out.push(plc);
out.push(Object.assign(plc, await prepareObject("plc", plc)));
}
ctx.response.body = out;
perf(ctx);
})
.get("/plc/:host", async (ctx) => {
const item = ats.ecosystem.data['plc-directories'].find((f) =>
f.url === `https://${ctx.params.host}`
);
if (!item) {
return ctx.response.code = 404;
}
Object.assign(item, await prepareObject("plc", item));
ctx.response.body = item;
perf(ctx);
})
.get("/_metrics", async (ctx) => {
const metrics = {};
@ -449,7 +560,7 @@ router
if (!item) {
return ctx.status = 404;
}
Object.assign(item, prepareObject("did", item));
Object.assign(item, await prepareObject("did", item));
ctx.response.body = item;
perf(ctx);
});

Zobrazit soubor

@ -0,0 +1,47 @@
<script>
import { onMount, beforeUpdate, afterUpdate } from 'svelte';
export let src;
let loaded = false;
let failed = false;
let loading = false;
let last = src;
let img;
beforeUpdate(() => {
if (src !== last) {
img.src = src;
last = src;
loading = true;
loaded = false;
console.log('loading again');
}
});
function processImage() {
img = new Image();
img.src = src;
loading = true;
img.onload = () => {
loading = false;
console.log('loaded');
loaded = true;
};
img.onerror = () => {
loading = false;
failed = true;
};
}
onMount(() => {
processImage();
});
</script>
{#if loaded}
<img {src} class="ratio-square w-full h-full rounded-full object-contains" />
{:else if loading}
<div class="bg-white/20 w-full h-full rounded-full" />
{/if}

Zobrazit soubor

@ -0,0 +1,157 @@
<script>
import {
TabGroup,
Tab,
TabAnchor,
ProgressRadial,
SlideToggle,
dataTableHandler
} from '@skeletonlabs/skeleton';
import Chart from '$lib/components/Chart.svelte';
import { formatNumber } from '$lib/utils';
import { writable } from 'svelte/store';
import { request } from '$lib/api';
import { onDestroy } from 'svelte';
export let title;
export let endpoint;
export let height = 'h-[40vh]';
export let processSeries = (data) =>
data ? [{ name: title, type: 'line', data: data.map((r) => r._value) }] : [];
export let unit = 'op/s';
export let valueFormatter = (val) => `${val !== undefined ? formatNumber(val) : '-'} ${unit}`;
export let axisDataProcess = (data) =>
data
.filter((r) => r.table === 0)
.map((r) => r._time)
.slice(1, -1) || [];
export let ranges = ['1h', '24h', '7d', '30d'];
export let chartType = null;
export let tabState = writable(0);
const chartActivityTab = tabState;
let chartActivity = null;
function renderActivityChart(chartData) {
let types = [];
let typesCount = {};
if (chartType === 'complex') {
for (const ci of chartData) {
const key = `${ci.mod}:${ci.type}`;
if (!types.includes(key)) {
types.push(key);
}
if (!typesCount[key]) {
typesCount[key] = 0;
}
typesCount[key] += ci._value;
}
}
types = types.sort((x, y) => (typesCount[x] < typesCount[y] ? 1 : -1));
return {
animationDuration: 250,
tooltip: {
trigger: 'axis',
valueFormatter
/*position: function (point, params, dom, rect, size) {
// fixed at top
return [point[0] + 50, point[1] + 50];
},*/
},
/*legend: {
data: types.map((c) => {
let [ mod, type ] = c.split(':')
return `${type.replace(/^app.bsky./, '')} (${mod})`
})
},*/
grid: {
left: '2%',
right: '2%',
bottom: '3%',
top: '10%',
containLabel: true
},
toolbox: {
feature: {
magicType: { show: true, type: ['stack', 'tiled'] },
saveAsImage: {}
}
},
xAxis: {
type: 'category',
//boundaryGap: false,
data: axisDataProcess(chartData)
},
yAxis: {
type: 'value',
axisLabel: {
formatter: `{value} ${unit}`
}
},
series: processSeries(chartData, types)
/*||chartType === 'complex' ?
types.map()
: [{
name: item.host,
type: 'line',
data: chartData.map((r) => r._value) || []
}]*/
};
}
let chartInterval = null;
chartActivityTab.subscribe(async (num) => {
if (!endpoint) {
return null;
}
chartActivity = null;
if (chartInterval) {
clearInterval(chartInterval);
}
async function render() {
const data = await request(
fetch,
`${endpoint}?range=${ranges[num]}&complex=${chartType === 'complex' ? 1 : ''}`
);
if (data) {
chartActivity = renderActivityChart(data.data);
}
}
render();
chartInterval = setInterval(() => {
render();
}, 15 * 1000);
});
onDestroy(() => {
if (chartInterval) {
clearInterval(chartInterval);
}
});
</script>
<h2 class="h2">{title}</h2>
<TabGroup>
<Tab bind:group={$chartActivityTab} name="tab1" value={0}>Last 1h</Tab>
<Tab bind:group={$chartActivityTab} name="tab2" value={1}>Last 24h</Tab>
<Tab bind:group={$chartActivityTab} name="tab3" value={2}>Last 7d</Tab>
<Tab bind:group={$chartActivityTab} name="tab4" value={3}>Last 30d</Tab>
<!-- Tab Panels --->
<svelte:fragment slot="panel">
<div class="w-full {height}">
{#if chartActivity}
<Chart options={chartActivity} />
{:else}
<div class="flex items-center justify-center w-full h-full">
<div>
<ProgressRadial />
</div>
</div>
{/if}
</div>
</svelte:fragment>
</TabGroup>

Zobrazit soubor

@ -9,7 +9,9 @@
pds: { url: `${data.config.api}/pds/%`, key: 'host' },
did: { url: `${data.config.api}/%`, key: 'did' },
client: { url: `${data.config.api}/client/%`, key: 'id' },
fed: { url: `${data.config.api}/fed/%`, key: 'id' }
fed: { url: `${data.config.api}/fed/%`, key: 'id' },
bgs: { url: `${data.config.api}/bgs/%`, key: 'host' },
plc: { url: `${data.config.api}/plc/%`, key: 'host' }
};
const config = models[model];
const url = config.url.replace('%', data.item[config.key]);

Zobrazit soubor

@ -1,6 +1,6 @@
<script>
import { blobUrl } from '$lib/utils';
import { is_empty } from 'svelte/internal';
import Avatar from '$lib/components/Avatar.svelte';
export let items;
export let data;
@ -26,12 +26,11 @@
<div class="flex gap-4 bg-surface-500/10 p-4" id={item.did}>
<div class="w-20 h-20 shrink-0">
<a href="/{item.did}" class="w-full h-full">
<img
<Avatar
id="image-{item.did}"
src={item.repo?.profile?.avatar?.ref?.$link
? blobUrl(item.did, item.repo?.profile?.avatar?.ref?.$link)
: '/avatar.svg'}
class="aspect-square object-cover rounded-full w-full h-full"
/>
</a>
</div>

Zobrazit soubor

@ -6,6 +6,7 @@ import { filesize as _filesize } from 'filesize';
import { config } from '$lib/config';
export function dateDistance(date) {
console.log('xxx', date);
return formatDistanceToNow(new Date(date));
}
@ -14,11 +15,12 @@ export function identicon(...args) {
}
numbro.setDefaults({
thousandSeparated: true
//mantissa: 2
thousandSeparated: true,
mantissa: 2,
trimMantissa: true
});
export function formatNumber(number) {
return numbro(number).format();
export function formatNumber(number, ...args) {
return numbro(number).format(...args);
}
export function customTableMapper(source, keys, process) {

Zobrazit soubor

@ -40,7 +40,9 @@
[
{ title: 'DIDs', url: '/dids' },
{ title: 'PDS Instances', url: '/pds' },
{ title: 'Federations', url: '/feds' }
{ title: 'BGS Instances', url: '/bgs' },
{ title: 'PLC Directories', url: '/plcs' }
//{ title: 'Federations', url: '/feds' }
],
[
{ title: 'API', url: '/api' },
@ -145,17 +147,33 @@
href="/pds"
class="btn hover:variant-soft-primary"
class:bg-primary-active-token={$page.url.pathname.startsWith('/pds')}
><span>{$i18n.t('PDS Instances')}</span></a
><span>{$i18n.t('PDS')}</span></a
>
</div>
<div class="relative hidden lg:block">
<a
href="/bgs"
class="btn hover:variant-soft-primary"
class:bg-primary-active-token={$page.url.pathname === '/bgs' ||
$page.url.pathname.startsWith('/bgs/')}><span>BGS</span></a
>
</div>
<div class="relative hidden lg:block">
<a
href="/plcs"
class="btn hover:variant-soft-primary"
class:bg-primary-active-token={$page.url.pathname === '/plcs' ||
$page.url.pathname.startsWith('/plcs/')}><span>PLC</span></a
>
</div>
<!--div class="relative hidden lg:block">
<a
href="/feds"
class="btn hover:variant-soft-primary"
class:bg-primary-active-token={$page.url.pathname === '/feds' ||
$page.url.pathname.startsWith('/fed/')}><span>Federations</span></a
>
</div>
</div-->
<!--div class="relative hidden lg:block">
<a
href="/clients"

Zobrazit soubor

@ -0,0 +1,7 @@
import { request } from '$lib/api';
export async function load({ fetch }) {
return {
bgs: request(fetch, '/bgs')
};
}

Zobrazit soubor

@ -0,0 +1,38 @@
<script>
import Table from '$lib/components/Table.svelte';
import { dataTableHandler, tableMapperValues, tableSourceValues } from '@skeletonlabs/skeleton';
import { dateDistance, formatNumber, customTableMapper } from '$lib/utils.js';
import BasicPage from '$lib/components/BasicPage.svelte';
export let data;
function tableMap({ val, key, row }) {
if (key === 'world') {
val = `<span class="badge variant-filled bg-ats-fed-${row.id} dark:bg-ats-fed-${row.id} opacity-70 text-white dark:text-black ucfirst">${row.id}</span>`;
}
if (key === 'host') {
val = `<span class="text-xl font-semibold">${row.url.replace(/^https?:\/\//, '')}</span>`;
}
if (key === 'url_raw') {
val = `/bgs/${row.host}`;
}
if (key === 'ops') {
val = row.status?.ops
? formatNumber(row.status?.ops, { trimMantissa: true, mantissa: 2 }) + ' op/s'
: '-';
}
return val;
}
const sourceData = data.bgs;
const tableSimple = {
// A list of heading labels.
head: ['Federation', 'Host', 'Activity'],
body: customTableMapper(sourceData, ['world', 'host', 'ops'], tableMap),
meta: customTableMapper(sourceData, ['id', 'url_raw'], tableMap)
};
</script>
<BasicPage {data} title="BGS Instances">
<Table source={tableSimple} />
</BasicPage>

Zobrazit soubor

@ -0,0 +1,7 @@
import { request } from '$lib/api';
export async function load({ params, fetch }) {
return {
item: request(fetch, `/bgs/${params.host}`)
};
}

Zobrazit soubor

@ -0,0 +1,58 @@
<script>
import Breadcrumb from '$lib/components/Breadcrumb.svelte';
import SourceSection from '$lib/components/SourceSection.svelte';
import BasicPage from '$lib/components/BasicPage.svelte';
import BasicChart from '$lib/components/BasicChart.svelte';
import { writable } from 'svelte/store';
export let data;
const tabState = writable(0);
function activityMapSeries(data, types) {
return types.map((key) => {
let [mod, type] = key.split(':');
return {
name: `${type.replace(/^app.bsky./, '')} (${mod})`,
type: 'line',
data:
data
.filter((i) => i.mod === mod && i.type === type)
.map((r) => r._value)
.slice(1, -1) || [],
lineStyle: {
normal: {
width: 1
}
},
stack: 'x',
areaStyle: {
opacity: 0.5
},
smooth: false
};
});
}
</script>
<BasicPage {data} title={data.item.host} breadcrumb={[{ label: 'BGS Instances', link: '/bgs' }]}>
<BasicChart
title="Activity"
endpoint="/bgs/{data.item.host}/ops"
chartType="complex"
processSeries={activityMapSeries}
{tabState}
/>
<BasicChart
title="Post latency"
endpoint="/bgs/{data.item.host}/postLatency"
unit="ms"
height="h-[20vh]"
{tabState}
/>
<SourceSection {data} model="bgs" hide="true" />
<div class="min-h-screen" />
</BasicPage>

Zobrazit soubor

@ -125,8 +125,8 @@
return {
animationDuration: 250,
tooltip: {
trigger: 'axis'
//formatter: '{b}: {c} ms'
trigger: 'axis',
valueFormatter: (val) => `${val !== undefined ? formatNumber(val) : '-'} ms`
},
legend: {
data: Object.keys(crawlers).map((c) => `${crawlers[c].region} (${crawlers[c].location})`)
@ -145,7 +145,11 @@
xAxis: {
type: 'category',
boundaryGap: false,
data: chartData.filter((r) => r.table === 0).map((r) => r._time) || []
data:
chartData
.filter((r) => r.table === 0)
.map((r) => r._time)
.slice(0, -1) || []
},
yAxis: {
type: 'value',
@ -159,7 +163,11 @@
name: `${crawlerOptions.region} (${crawlerOptions.location})`,
type: 'line',
//stack: 'ms',
data: chartData.filter((r) => r.crawler === crawler).map((r) => r._value) || []
data:
chartData
.filter((r) => r.crawler === crawler)
.map((r) => r._value)
.slice(0, -1) || []
};
})
};

Zobrazit soubor

@ -0,0 +1,7 @@
import { request } from '$lib/api';
export async function load({ params, fetch }) {
return {
item: request(fetch, `/plc/${params.host}`)
};
}

Zobrazit soubor

@ -0,0 +1,13 @@
<script>
import Breadcrumb from '$lib/components/Breadcrumb.svelte';
import SourceSection from '$lib/components/SourceSection.svelte';
import BasicPage from '$lib/components/BasicPage.svelte';
export let data;
const item = data.item;
</script>
<BasicPage {data} title={item.host} breadcrumb={[{ label: 'PLC Directories', link: '/plcs' }]}>
<SourceSection {data} model="plc" />
</BasicPage>

Zobrazit soubor

@ -0,0 +1,7 @@
import { request } from '$lib/api';
export async function load({ fetch }) {
return {
plcs: request(fetch, '/plcs')
};
}

Zobrazit soubor

@ -0,0 +1,52 @@
<script>
import Table from '$lib/components/Table.svelte';
import { dataTableHandler, tableMapperValues, tableSourceValues } from '@skeletonlabs/skeleton';
import { dateDistance, formatNumber, customTableMapper } from '$lib/utils.js';
import BasicPage from '$lib/components/BasicPage.svelte';
export let data;
function tableMap({ val, key, row }) {
console.log({ key, val });
if (key === 'world') {
val = `<span class="badge variant-filled bg-ats-fed-${row.federation} dark:bg-ats-fed-${row.federation} opacity-70 text-white dark:text-black ucfirst">${row.federation}</span>`;
}
if (key === 'host') {
val = `<span class="text-xl font-semibold">${val}</span>`;
}
if (key === 'didsCount') {
val = `<a href="/dids?q=fed:${row.federation}" class="anchor">${formatNumber(val)}</a>`;
}
if (key === 'pdsCount') {
val = `<a href="/pds?q=fed:${row.federation}" class="anchor">${
row.host === 'plc.directory' ? 1 : formatNumber(val)
}</a>`;
if (row.host === 'plc.directory') {
val += ` <div class="text-xs inline-block ml-2">(+${formatNumber(row.pdsCount)})</div>`;
}
}
if (key === 'time') {
val = row.lastUpdate ? `<span class="text-xs">${dateDistance(row.lastUpdate)} ago</a>` : '-';
}
if (key === 'url_raw') {
val = `/plc/${row.host}`;
}
return val;
}
const sourceData = data.plcs.sort((x, y) => (x.didsCount > y.didsCount ? -1 : 1));
const tableSimple = {
// A list of heading labels.
head: ['Federation', 'Host', 'DIDs', 'PDS', 'Last mod'],
body: customTableMapper(
sourceData,
['world', 'host', 'didsCount', 'pdsCount', 'time'],
tableMap
),
meta: customTableMapper(sourceData, ['id', 'url_raw'], tableMap)
};
</script>
<BasicPage {data} title="PLC Directories">
<Table source={tableSimple} />
</BasicPage>