prague-blockchain-week/utils/engine.js

import {
  copy,
  emptyDir,
  ensureDir,
  exists,
} from "https://deno.land/std@0.173.0/fs/mod.ts";
import { parse as tomlParse } from "https://deno.land/std@0.173.0/encoding/toml.ts";
import { load as yamlLoad } from "https://deno.land/x/js_yaml_port@3.14.0/js-yaml.js";
import { posix } from "https://deno.land/std@0.173.0/path/mod.ts";
import * as syncTools from "./sync.lib.js";
import format from "https://deno.land/x/date_fns@v2.22.1/format/index.js";
import addDays from "https://deno.land/x/date_fns@v2.22.1/addDays/index.ts";

let _silentMode = false;
const thumbSizes = [150, 300, 500];

export class DeConfEngine {
  constructor(options = {}) {
    this.options = options;
    this.tag = this.options.tag || "dev";
    this.srcDir = this.options.srcDir || "./data";
    this.outputDir = this.options.outputDir || "./dist";
    this.publicUrl = this.options.publicUrl || "https://data.prgblockweek.com";
    this.exploreUrl = this.options.exploreUrl ||
      "https://explore.prgblockweek.com";
    this.githubUrl = this.options.githubUrl ||
      "https://github.com/utxo-foundation/prague-blockchain-week/tree/main/data";

    if (options.silent) {
      _silentMode = true;
    }
  }
  async init() {
    this.entries = [];
    for await (const f of Deno.readDir(this.srcDir)) {
      if (!f.name.match(/^\d+$/)) continue;
      const pkg = new DeConf_Package(f.name, this);
      await pkg.load([this.srcDir, f.name]);
      this.entries.push(pkg);
    }
  }
  async build() {
    await emptyDir(this.outputDir);
    console.log(`Tag: ${this.tag}`);
    await _textWrite([this.outputDir, "TAG"], this.tag);
    for (const pkg of this.entries) {
      console.table(pkg.data.events.map((e) => e.data.index), ["name"]);
      await pkg.write(this.outputDir);
    }
    await _jsonWrite(
      [this.outputDir, "index.json"],
      this.entries.map((p) => ({
        id: p.id,
        name: p.data.index.name,
        dataUrl: p.data.index.dataUrl,
        exploreUrl: p.data.index.exploreUrl,
        __time: new Date(),
        __tag: this.tag,
      })),
    );
    // write schemas
    const schemaVersion = 1;
    const schemas = await this.schemas(schemaVersion);

    const outputSchemaDir = [this.outputDir, "schema", schemaVersion].join("/");
    await emptyDir(outputSchemaDir);
    console.log(`writing schema (v${schemaVersion}) ..`);

    const schemaBundle = {};
    for (const schema of schemas) {
      await _jsonWrite(
        [outputSchemaDir, schema.name + ".json"],
        schema.schema,
      );
      schemaBundle[schema.name] = schema.schema;
    }
    await _jsonWrite([outputSchemaDir, "bundle.json"], {
      definitions: schemaBundle,
    });
  }
  async schemas(version = "1") {
    const schemaDir = `./utils/schema/${version}`;
    const arr = [];
    for await (const f of Deno.readDir(schemaDir)) {
      const m = f.name.match(/^(.+)\.yaml$/);
      if (!m) {
        continue;
      }
      arr.push({
        name: m[1],
        schema: Object.assign(
          { $id: this.schemaUrl(version, m[1]) },
          await _yamlLoad([schemaDir, f.name].join("/")),
        ),
      });
    }
    return arr.sort((x, y) => x.name > y.name ? 1 : -1);
  }
  schemaUrl(version = "1", type = "index") {
    return `${this.publicUrl}/schema/${version}/${type}.json`;
  }
  entriesList() {
    return this.entries.map((e) => e.id);
  }
}

class DeConf_Package {
  constructor(id, engine) {
    this.id = id;
    this.data = null;
    this.engine = engine;
    this.tag = engine.tag;
    this.colMapper = {
      places: "place",
      events: "event",
      "media-partners": "media-partner",
      contributors: "contributor",
      benefits: "benefit",
      unions: "union",
      chains: "chain",
      "other-events": "event",
    };
    this.collections = Object.keys(this.colMapper);
  }

  async load(specDir) {
    const pkg = {};
    // load year index
    pkg.index = await _tomlLoad([...specDir, "index.toml"].join("/"));
    pkg.index.dataUrl = [this.engine.publicUrl, this.id].join("/");
    pkg.index.exploreUrl = [this.engine.exploreUrl, this.id].join("/");
    pkg.index.dataGithubUrl = [this.engine.githubUrl, this.id].join("/");
    //console.log(`\n##\n## [${pkg.index.name}] \n##`);
    // load sub-events

    for (const colPlural of this.collections) {
      pkg[colPlural] = await this.loadCollection(specDir, colPlural);
    }

    this.data = pkg;
  }
  async write(dir) {
    const outputDir = [dir, this.id].join("/");
    await emptyDir(outputDir);
    await this.assetsWrite(outputDir);
    await _jsonWrite([outputDir, "index.json"], this.toJSON());
  }
  async loadCollection(specDir, type) {
    const arr = [];
    for await (const ef of Deno.readDir([...specDir, type].join("/"))) {
      if (ef.name.match(/^_/)) continue;
      const m = ef.name.match(/^([\w\d\-]+)(\.toml|)$/);
      if (!m) continue;
      const ev = new DeConf_Collection(type, m[1], this.engine);
      try {
        await ev.load([...specDir, type, ef.name]);
      } catch (e) {
        throw new Error(`[item=${m[1]}]: ${e}`);
      }
      arr.push(ev);
    }
    return arr;
  }
  async assetsWrite(outputDir) {
    for (const colName of this.collections) {
      const dir = [outputDir, "assets", colName].join("/");
      await emptyDir(dir);
      for (const item of this.data[colName]) {
        await item.assetsWrite(
          dir,
          [this.engine.publicUrl, this.id, "assets", colName].join(
            "/",
          ),
        );
      }
    }
  }
  toJSON() {
    return Object.assign({ id: this.id }, this.data.index, {
      ...Object.fromEntries(
        Object.keys(this.colMapper).map((col) => {
          return [col, this.data[col]];
        }),
      ),
      __time: new Date(),
      __tag: this.tag,
    });
  }
}

class DeConf_Collection {
  constructor(type, id, engine) {
    this.type = type;
    this.id = id;
    this.data = null;
    this.dir = null;
    this.assets = ["logo", "photo"];
    this.haveSync = false;
    this.dataFile = null;
    this.engine = engine;
  }

  async load(path) {
    let fn;
    if (path[path.length - 1].match(/^(.+)\.toml$/)) {
      fn = path;
    } else {
      this.dir = path.join("/");
      fn = [...path, "index.toml"];
    }

    this.dataFile = [this.dir, "data.json"].join("/");

    const efIndex = await _tomlLoad(fn.join("/"));
    const hash = await _makeHash([this.type, this.id].join(":"));
    const data = {
      index: { id: this.id, hash, ...efIndex },
    };
    if (["events", "other-events"].includes(this.type)) {
      // add Event Segments
      if (!data.index.segments) {
        data.index.segments = [];
        for (let i = 0; i < data.index.days; i++) {
          data.index.segments.push({
            date: format(addDays(new Date(data.index.date), i), "yyyy-MM-dd"),
            times: data.index.times || "09:00-18:00",
          });
        }
      }
      for (const sg of data.index.segments) {
        if (sg.remote) {
          continue;
        }
        const [sstart, send] = sg.times.split("-");
        sg.startTime = (new Date(`${sg.date}T${sstart}`)).toISOString();
        const endDate = send <= sstart
          ? format(addDays(new Date(sg.date), 1), "yyyy-MM-dd")
          : sg.date;
        sg.endTime = (new Date(`${endDate}T${send}`)).toISOString();
      }
    }
    if (
      this.dir &&
      (!data.index.hidden ||
        this.engine.options.hiddenAllowed &&
          this.engine.options.hiddenAllowed.includes("bitcoin-prague"))
    ) {
      const syncDataFn = [this.dir, "data.json"].join("/");
      if (await exists(syncDataFn)) {
        data.sync = await _jsonLoad(syncDataFn);
      }
      for (const asset of this.assets) {
        if (data.index[asset]) {
          const assetFn = [this.dir, data.index[asset]].join("/");
          if (!await exists(assetFn)) {
            throw new Error(`Asset not exists: ${assetFn}`);
          }
        }
      }
    }
    // check if sync file exists
    this.syncFile = [this.dir, "_sync.js"].join("/");
    if (await exists(this.syncFile)) {
      this.haveSync = true;
    }
    this.data = data;
  }

  async optimizeImages() {
    for (const as of this.assets) {
      if (this.data.index[as]) {
        await this.optimizeImageFile(this.data.index[as]);
      }
    }
    if (this.data.index?.speakers) {
      for (const s of this.data.index.speakers) {
        await this.optimizeImageFile(s.photo);
      }
    }
    if (this.data.sync?.speakers) {
      for (const s of this.data.sync.speakers) {
        await this.optimizeImageFile(s.photo);
      }
    }
  }

  async optimizeImageFile(fn) {
    if (!fn) {
      return null;
    }
    const src = [this.dir, fn].join("/");
    const extname = posix.extname(src);
    const dest = [this.dir, fn.replace(/\.([^\.]+)$/, ".op.webp")].join("/");
    if (!await exists(dest)) {
      if (extname === ".webp") {
        await _fileCopy(src, dest);
        console.log(`${dest} copied`);
        return true;
      }
      await _imageOptimalizedWrite(src, dest);
      console.log(`${dest} writed`);
    }
    const info = await _imageWebPInfo(dest);
    if (!info) {
      return null;
    }
    const longer = info.width > info.height ? "w" : "h";
    const ratio = longer === "h"
      ? (info.width / info.height)
      : (info.height / info.width);
    for (const sz of thumbSizes) {
      const pxs = longer === "h"
        ? [sz, Math.round(sz / ratio)]
        : [Math.round(sz / ratio), sz];
      //console.log(`size=${sz} px_orig=${[info.width, info.height]} px=${pxs}`)
      //console.log(info.width, info.height, ratio, sz, cheight)
      const szDest = [this.dir, fn.replace(/\.([^\.]+)$/, `-${sz}px.op.webp`)]
        .join("/");
      if (!await exists(szDest)) {
        await _imageOptimalizedWrite(dest, szDest, pxs);
        console.log(`${szDest} writed`);
      }
    }
  }

  async sync() {
    if (!this.haveSync) return null;
    if (!_silentMode) console.log(`syncing ${this.id} ..`);
    const module = await import("../" + this.syncFile);
    // data
    if (module.data) {
      const data = await module.data(syncTools);
      if (!JSON.stringify(data)) {
        return null;
      }

      if (data.speakers) {
        data.speakers = data.speakers.sort((x, y) => x.id > y.id ? 1 : -1);
        const photosDir = [this.dir, "photos"].join("/");
        await ensureDir(photosDir);
        for (const sp of data.speakers) {
          if (!sp.photoUrl) continue;
          const ext = await posix.extname(sp.photoUrl);
          const nameId = sp.id || sp.name.toLowerCase().replace(/ /g, "-");
          const dir = [photosDir, "speakers"].join("/");
          const ffn = (sp.id ? sp.id : nameId) + ext.replace(/\?.+$/, "");
          const fn = [dir, ffn].join("/");
          if (await exists(fn)) {
            sp.photo = ["photos", "speakers", ffn].join("/");
            continue;
          }
          await ensureDir(dir);
          const photoFetch = await fetch(sp.photoUrl);
          if (!photoFetch.body) {
            continue;
          }
          const tmpfile = [dir, ffn].join("/");
          const file = await Deno.open(tmpfile, { write: true, create: true });
          await photoFetch.body.pipeTo(file.writable);
          //await _imageOptimalizedWrite(tmpfile, fn);
          console.log(`${fn} writed`);

          sp.photo = ["photos", "speakers", ffn].join("/");
        }
      }
      await _jsonWrite(this.dataFile, data);
      this.data.sync = data;
    }
  }

  async assetsWrite(outputDir, publicUrl) {
    const x = { ...this.data.sync, ...this.data.index };

    const writeImage = async (fn, outDir, fnRename = null) => {
      const srcFile = [this.dir, fn].join("/");
      if (await exists(srcFile)) {
        const outFile = [outDir, fnRename || posix.basename(fn)].join("/");
        await _fileCopy(srcFile, outFile);
      }
    };
    const writeImageBundle = async (src, outDir) => {
      await ensureDir(outDir);
      //await writeImage(src, outDir)
      //await writeImage(src.replace(/\.(.+)$/, '.op.webp'), outDir, posix.basename(src).replace(/\.(.+)$/, `.webp`))
      await writeImage(
        src.replace(/\.(.+)$/, "-500px.op.webp"),
        outDir,
        posix.basename(src).replace(/\.(.+)$/, `.webp`),
      );
      for (const sz of thumbSizes) {
        await writeImage(
          src.replace(/\.(.+)$/, `-${sz}px.op.webp`),
          outDir,
          posix.basename(src).replace(/\.(.+)$/, `_${sz}px.webp`),
        );
      }
    };
    for (const asset of this.assets) {
      if (!x[asset]) continue;

      const outDir = [outputDir, this.id].join("/");
      await emptyDir(outDir);
      await writeImageBundle(x[asset], outDir);
      const fnOut = [this.id, x[asset].replace(/\.[^.]+$/, ".webp")].join("/");
      const url = [publicUrl, fnOut].join("/");
      this.data.index[asset] = url;
    }

    const speakersCol = this.data.sync
      ? this.data.sync.speakers
      : this.data.index.speakers;
    if (speakersCol) {
      const outDir = [outputDir, this.id, "photos", "speakers"].join("/");
      for (const sp of speakersCol) {
        if (!sp.photo) continue;
        await writeImageBundle(sp.photo, outDir);

        sp.photoUrl = [
          publicUrl,
          this.id,
          "photos",
          "speakers",
          posix.basename(sp.photo).replace(/\.[^.]+$/, ".webp"),
        ].join("/");
      }
    }
  }

  opResolve(fn) {
    return [fn.replace(/[^\.]+$/, "op.webp"), fn.replace(/[^\.]+$/, "webp")];
  }

  toJSON() {
    return Object.assign({ id: this.id }, this.data.index, this.data.sync);
  }
}

async function _imageWebPInfo(src) {
  const p = Deno.run({
    cmd: ["webpinfo", src],
    stdout: "piped",
    stderr: "piped",
  });
  await p.status();
  const info = new TextDecoder().decode(await p.output());
  if (!info.trim()) {
    console.log(src);
    return null;
  }
  if (info.match("Errors detected.")) {
    return false
  }
  return {
    size: Number(info.match(/File size:\s+(\d+)/)[1]),
    width: Number(info.match(/Width: (\d+)/)[1]),
    height: Number(info.match(/Height: (\d+)/)[1]),
  };
}

async function _imageOptimalizedWrite(src, dest, resize = null) {
  const cmd = [
    "cwebp",
    ...(resize ? ["-resize", resize[0], resize[1]] : []),
    "-q",
    "80",
    src,
    "-o",
    dest,
  ];
  const p = Deno.run({ cmd, stdout: "piped", stderr: "piped" });
  await p.status();
  //const err = new TextDecoder().decode(await p.stderrOutput())
  //console.log(err)
}

async function _fileCopy(from, to) {
  await copy(from, to, { overwrite: true });
  if (!_silentMode) {
    console.log(`${from} copied to ${to}`);
  }
  return true;
}
async function _tomlLoad(fn) {
  return tomlParse(await Deno.readTextFile(fn));
}
async function _yamlLoad(fn) {
  return yamlLoad(await Deno.readTextFile(fn));
}
async function _jsonWrite(fn, data) {
  if (Array.isArray(fn)) {
    fn = fn.join("/");
  }
  await Deno.writeTextFile(fn, JSON.stringify(data, null, 2));
  if (!_silentMode) {
    console.log(`${fn} writed`);
  }
  return true;
}
async function _textWrite(fn, text) {
  if (Array.isArray(fn)) {
    fn = fn.join("/");
  }
  await Deno.writeTextFile(fn, text);
}
async function _jsonLoad(fn) {
  return JSON.parse(await Deno.readTextFile(fn));
}

async function _makeHash(str) {
  return Array.from(
    new Uint8Array(
      await crypto.subtle.digest("SHA-256", (new TextEncoder()).encode(str)),
    ),
  ).map((b) => b.toString(16).padStart(2, "0")).join("");
}