update firehose, users page

This commit is contained in:
tree 2023-07-09 11:43:42 +00:00
rodič c71d48bdf2
revize 36292e9eda
14 změnil soubory, kde provedl 319 přidání a 29 odebrání

Zobrazit soubor

@ -5,6 +5,13 @@ import { join } from "https://deno.land/std@0.192.0/path/posix.ts";
import { exists } from "https://deno.land/std@0.193.0/fs/exists.ts";
import { ensureDir } from "https://deno.land/std@0.192.0/fs/ensure_dir.ts";
import { Sha256 } from "https://deno.land/std@0.119.0/hash/sha256.ts";
//import imageDecode from "https://deno.land/x/wasm_image_decoder@v0.0.7/mod.js";
//import wasm_image_loader from 'npm:@saschazar/wasm-image-loader';
//import wasm_webp from 'npm:@saschazar/wasm-webp';
//import wasm_webp_options from 'npm:@saschazar/wasm-webp/options';
//const imageLoader = await wasm_image_loader();
//const webp = await wasm_webp();
//import * as zstd from "https://deno.land/x/zstd_wasm/deno/zstd.ts";
//await zstd.init();
@ -81,7 +88,7 @@ router
ctx.response.headers.set(headerKey, index.headers[headerKey]);
}
ctx.response.body = body;
perf(ctx);
//perf(ctx);
});
app.use(oakCors()); // Enable CORS for All Routes

Zobrazit soubor

@ -9,6 +9,7 @@ await ats.init();
ats.startDaemon();
const servers = ["local", "texas", "tokyo"];
const HTTP_PORT = ats.env.PORT || 6677;
if (Number(ats.env.PORT) === 6677) {
const didUpdatedSub = ats.nats.subscribe("ats.service.plc.did.*");
@ -53,10 +54,6 @@ if (Number(ats.env.PORT) === 6677) {
})();
}
const HTTP_PORT = ats.env.PORT || 6677;
const app = new Application();
const router = new Router();
function perf(ctx) {
if (ctx.request.url.toString().startsWith("http://localhost:")) {
return null;
@ -135,6 +132,12 @@ function prepareObject(type, item) {
break;
case "did":
item.current = item.revs && item.revs.length > 0
? item.revs[item.revs.length - 1]
: null;
item.handle = item.current && item.current.operation?.alsoKnownAs
? item.current.operation?.alsoKnownAs[0]?.replace(/^at:\/\//, "")
: null;
item.srcHost = item.src.replace(/^https?:\/\//, "");
item.fed = findDIDFed(item);
break;
@ -142,6 +145,9 @@ function prepareObject(type, item) {
return item;
}
const app = new Application();
const router = new Router();
router
.get("/", (ctx) => {
ctx.response.body = "ATScan API";
@ -198,7 +204,6 @@ router
perf(ctx);
})
.get("/dids", async (ctx) => {
const out = [];
const query = { $and: [{}] };
const availableSort = {
@ -225,6 +230,7 @@ router
: { lastMod: -1 };
let q = ctx.request.url.searchParams.get("q")?.replace(/^@/, "");
let searchInfo = null;
if (q) {
query.$and[0].$or = [];
const tokens = q.split(" ");
@ -250,10 +256,37 @@ router
}
const text = textArr.join(" ").trim();
if (text) {
query.$and[0].$or.push({ did: { $regex: text } });
const tsRes = await fetch(
"http://localhost:8108/multi_search?x-typesense-api-key=Kaey9ahMo7xoob1haivaithe2Aighoo3azohl2Joo5Aemoh4aishoogugh3Oowim",
{
method: "post",
body: JSON.stringify({
searches: [
{
collection: "dids",
exhaustive_search: true,
facet_by: "",
highlight_full_fields: "did,handle,prevHandles,name,desc",
page: 1,
per_page: 12,
q: text,
query_by: "did,handle,prevHandles,name,desc",
sort_by: "",
},
],
}),
},
);
const ts = await tsRes.json();
const didHits = ts.results[0].hits.map((hit) => hit.document.did);
searchInfo = ts.results[0];
query.$and[0].$or.push({ did: { $in: didHits } });
/*query.$and[0].$or.push({ did: { $regex: text } });
query.$and[0].$or.push({
"revs.operation.alsoKnownAs": { $regex: text },
});
});*/
}
if (query.$and[0].$or.length === 0) {
delete query.$and[0].$or;
@ -269,8 +302,9 @@ router
//console.log(JSON.stringify(query, null, 2), { sort, limit });
//console.log(JSON.stringify({ query, sort, inputSort, inputSortConfig }));
const count = await ats.db.did.count(query);
let count = await ats.db.did.count(query);
let out = [];
for (
const did
of (await ats.db.did.find(query).sort(sort).limit(limit).toArray())
@ -279,6 +313,17 @@ router
out.push(did);
}
if (searchInfo?.hits) {
out = out.map((item) => {
const si = searchInfo.hits.find((h) => h.document.did === item.did);
item.searchSort = searchInfo.hits.indexOf(si);
return item;
});
out = out.sort((x, y) => x.searchSort > y.searchSort ? 1 : -1);
count = searchInfo.found;
}
ctx.response.headers.append("X-Total-Count", count);
ctx.response.body = ctx.request.headers.get("x-ats-wrapped") === "true"
? { count, items: out }
@ -404,7 +449,7 @@ router
if (!item) {
return ctx.status = 404;
}
item.fed = findDIDFed(item);
Object.assign(item, prepareObject("did", item));
ctx.response.body = item;
perf(ctx);
});

Zobrazit soubor

@ -8,18 +8,35 @@ import { ATScan } from "./lib/atscan.js";
import * as atprotoApi from "npm:@atproto/api";
const { AppBskyActorProfile } = atprotoApi.default;
const HTTP_PORT = "6990";
const ats = new ATScan({ enableQueues: true });
ats.debug = true;
await ats.init();
import { Application, Router } from "https://deno.land/x/oak/mod.ts";
const app = new Application();
const router = new Router();
const counters = {};
const client = subscribeRepos(`wss://bsky.social`, { decodeRepoOps: true });
client.on("message", (m) => {
if (ComAtprotoSyncSubscribeRepos.isHandle(m)) {
console.log("handle", m);
}
if (ComAtprotoSyncSubscribeRepos.isCommit(m)) {
//console.log(m)
m.ops.forEach(async (op) => {
if (!counters[op.action]) {
counters[op.action] = {};
}
if (op.payload?.$type) {
if (!counters[op.action][op.payload.$type]) {
counters[op.action][op.payload.$type] = 0;
}
counters[op.action][op.payload.$type]++;
}
if (op.payload?.$type === "app.bsky.actor.profile") {
if (AppBskyActorProfile.isRecord(op.payload)) {
const did = m.repo;
@ -40,3 +57,22 @@ client.on("message", (m) => {
});
}
});
router
.get("/counters", (ctx) => {
ctx.response.body = counters;
})
.get("/_metrics", (ctx) => {
ctx.response.body = Object.keys(counters).map((mod) => {
return Object.keys(counters[mod]).map((type) => {
const val = counters[mod][type];
return `firehose_event{mod="${mod}",type="${type}"} ${val}`;
}).filter((v) => v.trim()).join("\n");
}).join("\n") + "\n";
});
app.use(router.routes());
app.listen({ port: HTTP_PORT });
console.log(`ATScan Firehose metrics API started at port ${HTTP_PORT}`);

Zobrazit soubor

@ -0,0 +1,55 @@
<script>
import { blobUrl } from '$lib/utils';
import { is_empty } from 'svelte/internal';
export let items;
export let data;
let type = null; // = 'grid';
</script>
{#if items}
<div class="grid grid-cols-1 gap-4">
{#each items as item}
{#if type === 'grid'}
<div>
<a href="/{item.did}">
<img
src={item.repo?.profile?.avatar?.ref?.$link
? blobUrl(item.did, item.repo?.profile?.avatar?.ref?.$link)
: '/avatar.svg'}
class="aspect-square object-cover"
/>
</a>
</div>
{:else}
<div class="flex gap-4 bg-surface-500/10 p-4" id={item.did}>
<div class="w-20 h-20 shrink-0">
<a href="/{item.did}" class="w-full h-full">
<img
id="image-{item.did}"
src={item.repo?.profile?.avatar?.ref?.$link
? blobUrl(item.did, item.repo?.profile?.avatar?.ref?.$link)
: '/avatar.svg'}
class="aspect-square object-cover rounded-full w-full h-full"
/>
</a>
</div>
<div>
<div class="text-xl font-semibold">
<a href="/{item.did}">{item.repo?.profile?.displayName || item.handle}</a>
{#if item.fed !== 'bluesky'}
<span
class="badge variant-filled bg-ats-fed-sandbox dark:bg-ats-fed-sandbox opacity-70 align-[0.3em] ml-2"
>{item.fed}</span
>
{/if}
</div>
<div><a href="https://bsky.app/profile/{item.handle}">@{item.handle}</a></div>
<div class="text-sm mt-2 opacity-70">{item.repo?.profile?.description || ''}</div>
</div>
</div>
{/if}
{/each}
</div>
{/if}

Zobrazit soubor

@ -1,4 +1,4 @@
import { connect as NATSConnect, StringCodec, JSONCodec } from 'nats.ws';
import { connect as NATSConnect, JSONCodec, StringCodec } from 'nats.ws';
import { writable } from 'svelte/store';
export let connected = writable(null);

Zobrazit soubor

@ -3,6 +3,7 @@ import { minidenticon } from 'minidenticons';
import { tableSourceValues } from '@skeletonlabs/skeleton';
import numbro from 'numbro';
import { filesize as _filesize } from 'filesize';
import { config } from '$lib/config';
export function dateDistance(date) {
return formatDistanceToNow(new Date(date));
@ -71,3 +72,7 @@ export function getPDSStatus(row) {
return { color, ico, text };
}
export function blobUrl(did, cid) {
return `${config.blobApi}/${did}/${cid}`;
}

Zobrazit soubor

@ -124,12 +124,20 @@
></a
>
<div class="lg:ml-8 flex gap-1">
<!--div class="relative hidden lg:block">
<a
href="/users"
class="btn hover:variant-soft-primary"
class:bg-primary-active-token={$page.url.pathname.startsWith('/users')}
><span>{$i18n.t('Users')}</span></a
>
</div-->
<div class="relative hidden lg:block">
<a
href="/dids"
class="btn hover:variant-soft-primary"
class:bg-primary-active-token={$page.url.pathname.startsWith('/dids')}
><span>{$i18n.t('DIDs')}</span></a
class:bg-primary-active-token={$page.url.pathname.startsWith('/dids') ||
$page.url.pathname.startsWith('/did:plc:')}><span>{$i18n.t('DIDs')}</span></a
>
</div>
<div class="relative hidden lg:block">
@ -144,8 +152,8 @@
<a
href="/feds"
class="btn hover:variant-soft-primary"
class:bg-primary-active-token={$page.url.pathname === '/feds'}
><span>Federations</span></a
class:bg-primary-active-token={$page.url.pathname === '/feds' ||
$page.url.pathname.startsWith('/fed/')}><span>Federations</span></a
>
</div>
<!--div class="relative hidden lg:block">

Zobrazit soubor

@ -1,5 +1,12 @@
<script>
import { dateDistance, identicon, getDIDProfileUrl, filesize, formatNumber } from '$lib/utils.js';
import {
dateDistance,
identicon,
getDIDProfileUrl,
filesize,
formatNumber,
blobUrl
} from '$lib/utils.js';
import { Table } from '@skeletonlabs/skeleton';
import { tableMapperValues, tableSourceValues } from '@skeletonlabs/skeleton';
import SourceSection from '$lib/components/SourceSection.svelte';
@ -195,7 +202,7 @@
<th class="text-right">Avatar</th>
<td
><img
src={`${data.config.blobApi}/${item.did}/${item.repo.profile.avatar.ref.$link}`}
src={blobUrl(item.did, item.repo.profile.avatar.ref.$link)}
class="w-40"
/></td
>
@ -206,7 +213,7 @@
<th class="text-right">Banner</th>
<td
><img
src={`${data.config.blobApi}/${item.did}/${item.repo.profile.banner.ref.$link}`}
src={blobUrl(item.did, item.repo.profile.banner.ref.$link)}
class="w-40"
/></td
>

Zobrazit soubor

@ -13,6 +13,7 @@ export async function load({ fetch, url, parent }) {
if (sort) {
args.push(`sort=${sort}`);
}
const res = await fetch(`${config.api}/dids` + (args.length > 0 ? '?' + args.join('&') : ''), {
headers: { 'x-ats-wrapped': 'true' }
});

Zobrazit soubor

@ -193,7 +193,7 @@
{:else}
<div class="text-xl">
{#if $search && $search?.trim() !== ''}
Search for <code class="code text-2xl variant-tertiary">{$search.trim()}</code>
Search for <code class="code text-xl variant-tertiary">{$search.trim()}</code>
{#if onlySandbox}(only sandbox){/if} ({formatNumber(totalCount)}):
{:else}
All DIDs {#if onlySandbox} on sandbox{/if} ({formatNumber(totalCount)}):

Zobrazit soubor

@ -190,15 +190,6 @@
</script>
<BasicPage {data} title="PDS Instances">
{#if $preferences.favoritePDS.length > 0}
<h2 class="h2">Your favourites</h2>
<PDSTable sourceData={favoritesData} {data} on:favoriteClick={(e) => onFavoriteClick(e)} />
<h2 class="h2">All instances</h2>
{/if}
<PDSMap data={baseData} />
<form on:submit|preventDefault={formSubmit} class="flex gap-4">
<input
class="input"
@ -209,6 +200,16 @@
/>
<!--button type="submit" class="btn variant-filled">Search</button-->
</form>
{#if !$search}
{#if $preferences.favoritePDS.length > 0}
<h2 class="h2">Your favourites</h2>
<PDSTable sourceData={favoritesData} {data} on:favoriteClick={(e) => onFavoriteClick(e)} />
<h2 class="h2">All instances</h2>
{/if}
{/if}
<div class="text-xl">
{#if $search && $search?.trim() !== ''}
Search for <code class="code text-2xl variant-tertiary">{$search.trim()}</code>
@ -216,6 +217,10 @@
{:else}
All PDS Instances ({formatNumber(sourceData.length)}):
{/if}
{#if !$search}
<PDSMap data={baseData} />
{/if}
</div>
<div class="min-h-screen">
<PDSTable

Zobrazit soubor

@ -0,0 +1,31 @@
import { request } from '$lib/api';
/** @type {import('./$types').PageLoad} */
export async function load({ fetch, url, parent }) {
let q = url.searchParams.get('q');
let sort = url.searchParams.get('sort');
if (!q && !sort) {
sort = '!size';
}
const args = ['limit=15'];
if (q) {
args.push(`q=${q}`);
}
if (sort) {
args.push(`sort=${sort}`);
}
const res = await request(fetch, '/dids' + (args.length > 0 ? '?' + args.join('&') : ''), {
headers: { 'x-ats-wrapped': 'true' }
});
//console.log(res.totalCount)
return {
items: res.items,
totalCount: res.count,
q,
sort
};
}

Zobrazit soubor

@ -0,0 +1,89 @@
<script>
import BasicPage from '$lib/components/BasicPage.svelte';
import UserList from '$lib/components/UserList.svelte';
import { writable } from 'svelte/store';
import { page } from '$app/stores';
import { goto, invalidate } from '$app/navigation';
import { ProgressRadial, SlideToggle } from '@skeletonlabs/skeleton';
import { formatNumber } from '$lib/utils';
import { beforeUpdate, afterUpdate } from 'svelte';
export let data;
$: totalCount = data.totalCount;
$: sourceData = data.items;
let search = writable(data.q || '');
$: mysearch = $search;
let sort = data.sort || null;
function formSubmit() {}
search.subscribe((val) => {
if (val?.trim() === data.q) {
return val;
}
//sourceData = null;
totalCount = null;
//onsole.log('xx')
gotoNewTableState();
return val;
});
function gotoNewTableState() {
let q = $search || '';
q = q.trim();
let args = [];
if (q) {
args.push(`q=${q}`);
}
/*if (sort && !(args.length === 0 && sort === '!size')) {
args.push(`sort=${sort}`);
}*/
const path = '/users' + (args.length > 0 ? '?' + args.join('&') : '');
const currentPath = $page.url.pathname + $page.url.search;
console.log(currentPath, path);
if (currentPath === path) {
return null;
}
goto(path, { keepFocus: true, noScroll: true });
}
</script>
<BasicPage {data} title="Users">
<form on:submit|preventDefault={formSubmit} class="flex gap-4">
<div class="flex w-full gap-4 items-center justify-center">
<div class="grow">
<input
class="input"
title="Input (text)"
type="text"
placeholder="Search for user .."
bind:value={$search}
autocomplete="off"
spellcheck="false"
autocorrect="off"
/>
</div>
</div>
<!--button type="submit" class="btn variant-filled">Search</button-->
</form>
<div class="min-h-screen">
<div class="text-xl mb-4">
{#if $search && $search?.trim() !== ''}
Search for <code class="code text-xl variant-tertiary">{$search.trim()}</code>
{#if totalCount !== null}({formatNumber(totalCount)}){/if}:
{:else}
All Users {#if totalCount !== null}({formatNumber(totalCount)}){/if}:
{/if}
</div>
{#if sourceData === null}
<!--div class="flex justify-center items-center w-full h-full">
<ProgressRadial />
</div!-->
{:else}
<UserList {data} items={sourceData} />
{/if}
</div>
</BasicPage>

Zobrazit soubor

@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" width="80" height="80" viewBox="0 0 24 24" fill="none" stroke="none" data-testid="userAvatarFallback"><circle cx="12" cy="12" r="12" fill="#0070ff"></circle><circle cx="12" cy="9.5" r="3.5" fill="#fff"></circle><path stroke-linecap="round" stroke-linejoin="round" fill="#fff" d="M 12.058 22.784 C 9.422 22.784 7.007 21.836 5.137 20.262 C 5.667 17.988 8.534 16.25 11.99 16.25 C 15.494 16.25 18.391 18.036 18.864 20.357 C 17.01 21.874 14.64 22.784 12.058 22.784 Z"></path></svg>

Za

Šířka:  |  Výška:  |  Velikost: 516 B