Sync updates

.gitignore

@@ -1,2 +1,3 @@
 .DS_Store
-dist
+dist
+cache

utils/engine.js

@@ -218,20 +218,21 @@ class DeConf_Collection {
 
       if (data.speakers) {
         const photosDir = [this.dir, "photos"].join("/");
-        await emptyDir(photosDir);
+        await ensureDir(photosDir);
         for (const sp of data.speakers) {
           if (!sp.photoUrl) continue;
-          const photoFetch = await fetch(sp.photoUrl);
           const ext = await posix.extname(sp.photoUrl);
           const dir = [photosDir, "speakers"].join("/");
+          const ffn = (sp.id ? sp.id : nameId) + ext;
+          const fn = [dir, ffn].join("/");
+          if (await exists(fn)) continue;
           await ensureDir(dir);
           const nameId = sp.id || sp.name.toLowerCase().replace(/ /g, "-");
+          const photoFetch = await fetch(sp.photoUrl);
           if (photoFetch.body) {
-            const ffn = (sp.id ? sp.id : nameId) + ext;
-            const fn = [dir, ffn].join("/");
-            console.log(`${fn} writed`);
             const file = await Deno.open(fn, { write: true, create: true });
             await photoFetch.body.pipeTo(file.writable);
+            console.log(`${fn} writed`);
             sp.photo = ["photos", "speakers", ffn].join("/");
           }
         }

utils/syncTools.js

@@ -1,4 +1,7 @@
 import cheerio from "https://esm.sh/cheerio";
+import { ensureDir, exists } from "https://deno.land/std@0.173.0/fs/mod.ts";
+
+const CACHE_DIR = "./cache";
 
 export async function loadJSONUrl(url) {
   const resp = await fetch(url);
@@ -6,11 +9,33 @@ export async function loadJSONUrl(url) {
 }
 
 export async function loadHtmlUrl(url) {
+  const cacheTimeout = 3600;
+  await ensureDir(CACHE_DIR);
+  const hash = Array.from(
+    new Uint8Array(
+      await crypto.subtle.digest("SHA-256", (new TextEncoder()).encode(url)),
+    ),
+  ).map((b) => b.toString(16).padStart(2, "0")).join("");
+  const cacheFn = `${CACHE_DIR}/${hash}`;
+
+  if (await exists(cacheFn)) {
+    console.log(`Cache found! ${hash}`);
+    return cheerio.load(await Deno.readTextFile(cacheFn));
+  }
+
+  console.log(`Getting ${url}`);
   const resp = await fetch(url, {
     headers: {
       "User-Agent":
         "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.1 Safari/605.1.15",
     },
   });
-  return cheerio.load(await resp.text());
+  const output = await resp.text();
+  await Deno.writeTextFile(cacheFn, output);
+  return cheerio.load(output);
+}
+
+export async function loadHtmlLocal(fn) {
+  const text = cheerio.load(await Deno.readTextFile(fn));
+  return text;
 }