zrcadlo https://github.com/atscan/atscan
better handle verification
This commit is contained in:
rodič
0fedf949e5
revize
0981d5dc5c
|
|
@ -3,8 +3,11 @@ import { timeout } from "./utils.js";
|
|||
import { join } from "https://deno.land/std@0.192.0/path/posix.ts";
|
||||
import { ensureDir } from "https://deno.land/std@0.192.0/fs/ensure_dir.ts";
|
||||
import * as fsize from "npm:filesize";
|
||||
import * as zstd from 'https://deno.land/x/zstd_wasm/deno/zstd.ts';
|
||||
import { Sha256 } from "https://deno.land/std@0.119.0/hash/sha256.ts";
|
||||
|
||||
const filesize = fsize.filesize;
|
||||
await zstd.init();
|
||||
|
||||
export async function saveRepo(ats, didInfo, job = null) {
|
||||
if (didInfo.skipRepo) {
|
||||
|
|
@ -73,14 +76,23 @@ export async function saveRepo(ats, didInfo, job = null) {
|
|||
return;
|
||||
}
|
||||
|
||||
repo.carHash = new Sha256().update(data).hex();
|
||||
|
||||
// ensure db directory
|
||||
const dbPathBase = ats.env.ATSCAN_DB_PATH || "./db";
|
||||
const dbPath = join(dbPathBase, "repo");
|
||||
await ensureDir(dbPath);
|
||||
|
||||
// write car file
|
||||
const carFn = join(dbPath, `${did}.car`);
|
||||
await Deno.writeFile(carFn, data);
|
||||
//const carFn = join(dbPath, `${did}.car`);
|
||||
//await Deno.writeFile(carFn, data);
|
||||
|
||||
// write compressed version
|
||||
const compressed = zstd.compress(data);
|
||||
repo.sizeCompressed = compressed.length;
|
||||
repo.carCompressedHash = new Sha256().update(compressed).hex();
|
||||
const carFnCompressed = join(dbPath, `${did}.car.zst`);
|
||||
await Deno.writeFile(carFnCompressed, compressed);
|
||||
|
||||
// write index file
|
||||
const indexFn = join(dbPath, `${did}.json`);
|
||||
|
|
@ -96,7 +108,7 @@ export async function saveRepo(ats, didInfo, job = null) {
|
|||
await job.log(
|
||||
`[${did}@${pds}] displayName=${
|
||||
JSON.stringify(repo.profile?.displayName)
|
||||
} [${filesize(repo.size)}] ${carFn}`,
|
||||
} [${filesize(repo.size)}] compressed: ${carFnCompressed} [${filesize(repo.sizeCompressed)}]`,
|
||||
);
|
||||
await job.updateProgress(99);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ async function crawlUrl(ats, url, host = "local") {
|
|||
if (host === "local") {
|
||||
try {
|
||||
const [, ms] = await timeout(
|
||||
TIMEOUT,
|
||||
5000,
|
||||
fetch(url, {
|
||||
method: "OPTIONS",
|
||||
headers: {
|
||||
|
|
@ -38,7 +38,9 @@ async function crawlUrl(ats, url, host = "local") {
|
|||
},
|
||||
}),
|
||||
);
|
||||
if (res) {
|
||||
if (!res.ok) {
|
||||
err = res.status;
|
||||
} else {
|
||||
data = await res.json();
|
||||
}
|
||||
} catch (e) {
|
||||
|
|
@ -77,7 +79,7 @@ async function crawlUrl(ats, url, host = "local") {
|
|||
|
||||
async function crawl(ats) {
|
||||
const arr = await ats.db.pds.find().toArray();
|
||||
const results = pooledMap(25, arr.slice(0, 1000), async (i) => {
|
||||
const results = pooledMap(10, arr.slice(0, 1000), async (i) => {
|
||||
let err = null;
|
||||
|
||||
if (i.url.match(/^https?:\/\/(localhost|example.com)/)) {
|
||||
|
|
@ -89,8 +91,10 @@ async function crawl(ats) {
|
|||
}
|
||||
|
||||
const host = i.url.replace(/^https?:\/\//, "");
|
||||
const dontHaveA = i.dns &&
|
||||
(!i.dns.Answer || i.dns.Answer.filter((a) => a.type === 1).length === 0);
|
||||
|
||||
if (!i.dns) {
|
||||
if (!i.dns || dontHaveA) {
|
||||
console.log("sending dns request: ", i.url);
|
||||
let dns =
|
||||
await (await fetch(`https://dns.google/resolve?name=${host}&type=A`))
|
||||
|
|
|
|||
|
|
@ -78,9 +78,9 @@ async function traversePDSRepos(ats, item, cursor = null) {
|
|||
async function crawlNew(ats) {
|
||||
const pds = await ats.db.pds.find({}).toArray();
|
||||
const results = pooledMap(CONCURRENCY, _.shuffle(pds), async (item) => {
|
||||
if (item.url === "https://bsky.social") {
|
||||
/*if (item.url !== "https://bsky.social") {
|
||||
return null;
|
||||
}
|
||||
}*/
|
||||
if (!item.inspect.current || item.inspect.current.err) {
|
||||
return null;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "atscan-fe",
|
||||
"version": "0.7.1-alpha",
|
||||
"version": "0.7.2-alpha",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "vite dev",
|
||||
|
|
|
|||
|
|
@ -13,19 +13,32 @@
|
|||
const item = data.item;
|
||||
const asa = item.revs[item.revs.length - 1].operation?.alsoKnownAs;
|
||||
const handles = asa ? asa.map((h) => h.replace(/^at:\/\//, '')) : [];
|
||||
let verifiedHandles = null;
|
||||
let handleVerification = null;
|
||||
|
||||
export function checkDNSHandleUrl(host) {
|
||||
return `https://dns.google/resolve?name=_atproto.${host}&type=TXT`;
|
||||
}
|
||||
|
||||
async function checkDNSHandle(did, host) {
|
||||
const resp = await fetch(checkDNSHandleUrl(host));
|
||||
const out = await resp.json();
|
||||
if (!out.Answer) {
|
||||
return false;
|
||||
async function verifyHandle(handle, did) {
|
||||
const verifications = [];
|
||||
// firstly we try DNS style
|
||||
const dnsCheckUrl = `https://dns.google/resolve?name=_atproto.${handle}&type=TXT`;
|
||||
const dnsResp = await fetch(dnsCheckUrl);
|
||||
const dnsData = await dnsResp.json();
|
||||
verifications.push({
|
||||
verified:
|
||||
dnsData.Answer && dnsData.Answer.find((a) => a.type === 16 && a.data === `did=${did}`),
|
||||
type: 'dns',
|
||||
url: dnsCheckUrl
|
||||
});
|
||||
// if DNS verification failed, then we try http based
|
||||
if (!verifications[0].verified) {
|
||||
const httpCheckUrl = `https://${handle}/.well-known/atproto-did`;
|
||||
const httpResp = await fetch(httpCheckUrl);
|
||||
const httpString = await httpResp.text();
|
||||
verifications.push({
|
||||
verified: httpString.includes(did),
|
||||
type: 'http',
|
||||
url: httpCheckUrl
|
||||
});
|
||||
}
|
||||
return out.Answer.find((a) => a.type === 16 && a.data === `did=${did}`);
|
||||
return verifications;
|
||||
}
|
||||
|
||||
function tableMapperValuesLocal(source, keys) {
|
||||
|
|
@ -45,20 +58,27 @@
|
|||
: null;
|
||||
if (handle) {
|
||||
val = `@${handle}`;
|
||||
if (!handle.match(/\.bsky\.social$/) && i === source.length - 1) {
|
||||
if (i === source.length - 1) {
|
||||
let hstr = null;
|
||||
if (verifiedHandles === null) {
|
||||
console.log(handleVerification);
|
||||
if (handleVerification === null) {
|
||||
hstr =
|
||||
'<i class="fa-solid fa-clock opacity-30" alt="Loading .." title="Loading .."></i>';
|
||||
} else if (verifiedHandles.includes(handle)) {
|
||||
hstr =
|
||||
'<i class="fa-solid fa-circle-check text-green-500" alt="Verified" title="Verified"></i>';
|
||||
} else if (handleVerification && handleVerification.find((v) => v.verified)) {
|
||||
const method = handleVerification.find((v) => v.verified);
|
||||
hstr = `<i class="fa-solid fa-circle-check text-green-500" alt="Verified" title="Verified"></i> <span class="text-xs text-green-500 uppercase">${method.type}</span>`;
|
||||
} else {
|
||||
hstr =
|
||||
'<i class="fa-solid fa-circle-xmark text-red-500" alt="Not verified" title="Not verified"></i>';
|
||||
}
|
||||
if (hstr) {
|
||||
val += `<a href="${checkDNSHandleUrl(handle)}" class="ml-1.5">${hstr}</a>`;
|
||||
if (handleVerification) {
|
||||
val += `<a href="${
|
||||
handleVerification.find((v) => v.verified)?.url || handleVerification[0].url
|
||||
}" class="ml-3" target="_blank">${hstr}</a>`;
|
||||
} else {
|
||||
val += `<div class="inline-block ml-3">${hstr}</div>`;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -117,15 +137,9 @@
|
|||
currentError = e.message;
|
||||
}
|
||||
}
|
||||
if (handles && handles[0] && !handles[0].match(/\.bsky\.social$/)) {
|
||||
const verified = await checkDNSHandle(item.did, handles[0]);
|
||||
if (verified) {
|
||||
verifiedHandles = [handles[0]];
|
||||
historyTable = renderTable();
|
||||
} else {
|
||||
verifiedHandles = [];
|
||||
historyTable = renderTable();
|
||||
}
|
||||
if (handles && handles[0]) {
|
||||
handleVerification = await verifyHandle(handles[0], item.did);
|
||||
historyTable = renderTable();
|
||||
}
|
||||
});
|
||||
</script>
|
||||
|
|
@ -227,9 +241,18 @@
|
|||
</div></td
|
||||
>
|
||||
</tr>
|
||||
<tr>
|
||||
<th class="text-right">Hash</th>
|
||||
<td>{item.repo?.carHash || '-'}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th class="text-right">Size</th>
|
||||
<td>{filesize(item.repo?.size)}</td>
|
||||
<td
|
||||
>{filesize(item.repo?.size)}
|
||||
{#if item.repo?.sizeCompressed}
|
||||
({filesize(item.repo.sizeCompressed)} compressed)
|
||||
{/if}
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<th class="text-right">Records</th>
|
||||
|
|
|
|||
Načítá se…
Odkázat v novém úkolu