feat: add file exporter and progress

.gitignore

@@ -1,2 +1,4 @@
 *.json
 .vscode/
+export/
+cache/

cache.ts

@@ -0,0 +1,62 @@
+// deno-lint-ignore-file require-await
+import { path } from "./deps.ts";
+
+export type Cache = {
+  read: <T>(key: string) => Promise<T | undefined>;
+  write: <T>(key: string, value: T) => Promise<void>;
+};
+
+export class MemoryCache implements Cache {
+  private cache: Record<string, unknown> = {};
+
+  async read<T>(key: string): Promise<T | undefined> {
+    return this.cache[key] as T;
+  }
+
+  async write<T>(key: string, value: T): Promise<void> {
+    this.cache[key] = value;
+  }
+}
+
+/**
+ * File Cache stores data in a folder. Each file is named by the sha256 of its key.
+ */
+export class FileCache implements Cache {
+  constructor(private path: string) {}
+
+  private async getPath(key: string): Promise<string> {
+    await Deno.mkdir(this.path, { recursive: true });
+
+    const hash = await crypto.subtle.digest(
+      "SHA-256",
+      new TextEncoder().encode(key),
+    );
+    const hashHex = Array.from(new Uint8Array(hash))
+      .map((b) => b.toString(16).padStart(2, "0"))
+      .join("");
+
+    return path.join(this.path, hashHex);
+  }
+
+  async read<T>(key: string): Promise<T | undefined> {
+    const path = await this.getPath(key);
+    try {
+      const data = await Deno.readTextFile(path);
+      return JSON.parse(data);
+    } catch (e) {
+      if (e instanceof Deno.errors.NotFound) {
+        return undefined;
+      }
+      throw e;
+    }
+  }
+
+  async write<T>(key: string, value: T): Promise<void> {
+    const path = await this.getPath(key);
+    const encoder = new TextEncoder();
+    const data = encoder.encode(JSON.stringify(value));
+    const swapPath = `${path}.swap`;
+    await Deno.writeFile(swapPath, data);
+    await Deno.rename(swapPath, path);
+  }
+}

deps.ts

@@ -8,3 +8,7 @@ export * as flags from "https://deno.land/std@0.160.0/flags/mod.ts";
 export * as io from "https://deno.land/std@0.160.0/io/mod.ts";
 export * as uuid from "https://deno.land/std@0.160.0/uuid/mod.ts";
 export * as msgpack from "https://deno.land/x/msgpack@v1.4/mod.ts";
+export * as path from "https://deno.land/std@0.160.0/path/mod.ts";
+export * as datetime from "https://deno.land/std@0.160.0/datetime/mod.ts";
+export { MultiProgressBar } from "https://deno.land/x/progress@v1.2.8/mod.ts";
+export { Mutex } from "https://deno.land/x/semaphore@v1.1.1/mod.ts";

exporter/file.ts

@@ -0,0 +1,53 @@
+import { BattleExporter, VsHistoryDetail } from "../types.ts";
+import { datetime, path } from "../deps.ts";
+import { NSOAPP_VERSION, S3SI_VERSION } from "../constant.ts";
+
+const FILENAME_FORMAT = "yyyyMMddHHmmss";
+
+type FileExporterType = {
+  type: "VS" | "COOP";
+  nsoVersion: string;
+  s3siVersion: string;
+  exportTime: string;
+  data: VsHistoryDetail;
+};
+
+/**
+ * Exporter to file.
+ *
+ * This is useful for debugging. It will write each battle detail to a file.
+ * Timestamp is used as filename. Example: 2021-01-01T00:00:00.000Z.json
+ */
+export class FileExporter implements BattleExporter<VsHistoryDetail> {
+  name = "file";
+  constructor(private exportPath: string) {
+  }
+  async exportBattle(detail: VsHistoryDetail) {
+    await Deno.mkdir(this.exportPath, { recursive: true });
+
+    const playedTime = new Date(detail.playedTime);
+    const filename = `${datetime.format(playedTime, FILENAME_FORMAT)}.json`;
+    const filepath = path.join(this.exportPath, filename);
+
+    const body: FileExporterType = {
+      type: "VS",
+      nsoVersion: NSOAPP_VERSION,
+      s3siVersion: S3SI_VERSION,
+      exportTime: new Date().toISOString(),
+      data: detail,
+    };
+
+    await Deno.writeTextFile(filepath, JSON.stringify(body));
+  }
+  async getLatestBattleTime() {
+    const dirs: Deno.DirEntry[] = [];
+    for await (const i of Deno.readDir(this.exportPath)) dirs.push(i);
+
+    const files = dirs.filter((i) => i.isFile).map((i) => i.name);
+    const timestamps = files.map((i) => i.replace(/\.json$/, "")).map((i) =>
+      datetime.parse(i, FILENAME_FORMAT)
+    );
+
+    return timestamps.reduce((a, b) => (a > b ? a : b), new Date(0));
+  }
+}

exporter/stat.ink.ts

@@ -0,0 +1,21 @@
+import { BattleExporter, VsHistoryDetail } from "../types.ts";
+
+/**
+ * Exporter to stat.ink.
+ *
+ * This is the default exporter. It will upload each battle detail to stat.ink.
+ */
+export class StatInkExporter implements BattleExporter<VsHistoryDetail> {
+  name = "stat.ink";
+  constructor(private statInkApiKey: string) {
+    if (statInkApiKey.length !== 43) {
+      throw new Error("Invalid stat.ink API key");
+    }
+  }
+  async exportBattle(detail: VsHistoryDetail) {
+    throw new Error("Function not implemented.");
+  }
+  async getLatestBattleTime() {
+    return new Date();
+  }
+}

iksm.ts

@@ -58,7 +58,7 @@ export async function loginManually(): Promise<string> {
     'Log in, right click the "Select this account" button, copy the link address, and paste it below:',
   );
 
-  const login = await readline();
+  const login = (await readline()).trim();
   if (!login) {
     throw new Error("No login URL provided");
   }

s3si.ts

@@ -1,44 +1,103 @@
 import { getBulletToken, getGToken, loginManually } from "./iksm.ts";
 import { APIError } from "./APIError.ts";
-import { flags } from "./deps.ts";
+import { flags, MultiProgressBar, Mutex } from "./deps.ts";
 import { DEFAULT_STATE, State } from "./state.ts";
-import { checkToken, getBattleList } from "./splatnet3.ts";
+import { checkToken, getBattleDetail, getBattleList } from "./splatnet3.ts";
+import { BattleExporter, VsHistoryDetail } from "./types.ts";
+import { Cache, FileCache, MemoryCache } from "./cache.ts";
+import { StatInkExporter } from "./exporter/stat.ink.ts";
+import { readline } from "./utils.ts";
+import { FileExporter } from "./exporter/file.ts";
 
 type Opts = {
-  configPath: string;
+  profilePath: string;
+  exporter: string;
+  progress: boolean;
   help?: boolean;
 };
 
-const DEFAULT_OPTS = {
+const DEFAULT_OPTS: Opts = {
-  configPath: "./config.json",
+  profilePath: "./profile.json",
+  exporter: "stat.ink",
+  progress: true,
   help: false,
 };
 
+/**
+ * Fetch battle and cache it.
+ */
+class BattleFetcher {
+  state: State;
+  cache: Cache;
+  lock: Record<string, Mutex | undefined> = {};
+
+  constructor(
+    { cache = new MemoryCache(), state }: { state: State; cache?: Cache },
+  ) {
+    this.state = state;
+    this.cache = cache;
+  }
+  getLock(id: string): Mutex {
+    let cur = this.lock[id];
+    if (!cur) {
+      cur = new Mutex();
+      this.lock[id] = cur;
+    }
+    return cur;
+  }
+  fetchBattle(id: string): Promise<VsHistoryDetail> {
+    const lock = this.getLock(id);
+
+    return lock.use(async () => {
+      const cached = await this.cache.read<VsHistoryDetail>(id);
+      if (cached) {
+        return cached;
+      }
+
+      const detail = (await getBattleDetail(this.state, id))
+        .vsHistoryDetail;
+
+      await this.cache.write(id, detail);
+
+      return detail;
+    });
+  }
+}
+
+type Progress = {
+  current: number;
+  total: number;
+};
+
 class App {
   state: State = DEFAULT_STATE;
+
   constructor(public opts: Opts) {
     if (this.opts.help) {
       console.log(
         `Usage: deno run --allow-net --allow-read --allow-write ${Deno.mainModule} [options]
 
 Options:
-    --config-path <path>    Path to config file (default: ./config.json)
+    --profile-path <path>, -p    Path to config file (default: ./profile.json)
+    --exporter <exporter>, -e    Exporter to use (default: stat.ink), available: stat.ink,file
+    --no-progress, -n            Disable progress bar
     --help                       Show this help message and exit`,
       );
       Deno.exit(0);
     }
   }
-  async writeState() {
+  async writeState(newState: State) {
+    this.state = newState;
     const encoder = new TextEncoder();
     const data = encoder.encode(JSON.stringify(this.state, undefined, 2));
-    const swapPath = `${this.opts.configPath}.swap`;
+    const swapPath = `${this.opts.profilePath}.swap`;
     await Deno.writeFile(swapPath, data);
-    await Deno.rename(swapPath, this.opts.configPath);
+    await Deno.rename(swapPath, this.opts.profilePath);
   }
   async readState() {
     const decoder = new TextDecoder();
     try {
-      const data = await Deno.readFile(this.opts.configPath);
+      const data = await Deno.readFile(this.opts.profilePath);
       const json = JSON.parse(decoder.decode(data));
       this.state = {
         ...DEFAULT_STATE,
@@ -48,23 +107,60 @@ Options:
       console.warn(
         `Failed to read config file, create new config file. (${e})`,
       );
-      await this.writeState();
+      await this.writeState(DEFAULT_STATE);
     }
   }
+  async getExporters(): Promise<BattleExporter<VsHistoryDetail>[]> {
+    const exporters = this.opts.exporter.split(",");
+    const out: BattleExporter<VsHistoryDetail>[] = [];
+
+    if (exporters.includes("stat.ink")) {
+      if (!this.state.statInkApiKey) {
+        console.log("stat.ink API key is not set. Please enter below.");
+        const key = (await readline()).trim();
+        if (!key) {
+          console.error("API key is required.");
+          Deno.exit(1);
+        }
+        await this.writeState({
+          ...this.state,
+          statInkApiKey: key,
+        });
+      }
+      out.push(new StatInkExporter(this.state.statInkApiKey!));
+    }
+
+    if (exporters.includes("file")) {
+      out.push(new FileExporter(this.state.fileExportPath));
+    }
+
+    return out;
+  }
   async run() {
     await this.readState();
 
+    const bar = this.opts.progress
+      ? new MultiProgressBar({
+        title: "Export battles",
+      })
+      : undefined;
+    const exporters = await this.getExporters();
+
     try {
       if (!this.state.loginState?.sessionToken) {
         const sessionToken = await loginManually();
-        this.state.loginState = {
+
+        await this.writeState({
+          ...this.state,
+          loginState: {
             ...this.state.loginState,
             sessionToken,
-        };
+          },
-        await this.writeState();
+        });
       }
-      const sessionToken = this.state.loginState.sessionToken!;
+      const sessionToken = this.state.loginState!.sessionToken!;
 
+      console.log("Checking token...");
       if (!await checkToken(this.state)) {
         console.log("Token expired, refetch tokens.");
 
@@ -80,22 +176,51 @@ Options:
           appUserAgent: this.state.appUserAgent,
         });
 
-        this.state = {
+        await this.writeState({
           ...this.state,
           loginState: {
             ...this.state.loginState,
             gToken: webServiceToken,
             bulletToken,
           },
-          userLang,
+          userLang: this.state.userLang ?? userLang,
-          userCountry,
+          userCountry: this.state.userCountry ?? userCountry,
-        };
+        });
-
-        await this.writeState();
       }
 
+      const fetcher = new BattleFetcher({
+        cache: new FileCache(this.state.cacheDir),
+        state: this.state,
+      });
+      console.log("Fetching battle list...");
       const battleList = await getBattleList(this.state);
-      console.log(battleList);
+
+      const allProgress: Record<string, Progress> = Object.fromEntries(
+        exporters.map((i) => [i.name, {
+          current: 0,
+          total: 1,
+        }]),
+      );
+      const redraw = (name: string, progress: Progress) => {
+        allProgress[name] = progress;
+        bar?.render(
+          Object.entries(allProgress).map(([name, progress]) => ({
+            completed: progress.current,
+            total: progress.total,
+            text: name,
+          })),
+        );
+      };
+      await Promise.all(
+        exporters.map((e) =>
+          this.exportBattleList(
+            fetcher,
+            e,
+            battleList,
+            (progress) => redraw(e.name, progress),
+          )
+        ),
+      );
     } catch (e) {
       if (e instanceof APIError) {
         console.error(`APIError: ${e.message}`, e.response, e.json);
@@ -104,15 +229,56 @@ Options:
       }
     }
   }
+  /**
+   * Export battle list.
+   *
+   * @param fetcher BattleFetcher
+   * @param exporter BattleExporter
+   * @param battleList ID list of battles, sorted by date, newest first
+   * @param onStep Callback function called when a battle is exported
+   */
+  async exportBattleList(
+    fetcher: BattleFetcher,
+    exporter: BattleExporter<VsHistoryDetail>,
+    battleList: string[],
+    onStep?: (progress: Progress) => void,
+  ) {
+    const workQueue = battleList;
+    let done = 0;
+
+    const step = async (battle: string) => {
+      const detail = await fetcher.fetchBattle(battle);
+      await exporter.exportBattle(detail);
+      done += 1;
+      onStep?.({
+        current: done,
+        total: workQueue.length,
+      });
+    };
+
+    onStep?.({
+      current: done,
+      total: workQueue.length,
+    });
+    for (const battle of workQueue) {
+      await step(battle);
+    }
+  }
 }
 
 const parseArgs = (args: string[]) => {
   const parsed = flags.parse(args, {
-    string: ["configPath"],
+    string: ["profilePath", "exporter"],
-    boolean: ["help"],
+    boolean: ["help", "progress"],
+    negatable: ["progress"],
     alias: {
       "help": "h",
-      "configPath": ["c", "config-path"],
+      "profilePath": ["p", "profile-path"],
+      "exporter": ["e"],
+      "progress": ["n"],
+    },
+    default: {
+      progress: true,
     },
   });
   return parsed;

state.ts

@@ -9,8 +9,16 @@ export type State = {
   appUserAgent?: string;
   userLang?: string;
   userCountry?: string;
+
+  cacheDir: string;
+
+  // Exporter config
+  statInkApiKey?: string;
+  fileExportPath: string;
 };
 
 export const DEFAULT_STATE: State = {
+  cacheDir: "./cache",
   fGen: "https://api.imink.app/f",
+  fileExportPath: "./export",
 };

types.ts

@@ -37,6 +37,31 @@ export type HistoryGroups = {
     };
   }[];
 };
+export type VsHistoryDetail = {
+  id: string;
+  vsRule: {
+    name: string;
+    id: string;
+    rule: "TURF_WAR" | "AREA" | "LOFT" | "GOAL" | "CLAM" | "TRI_COLOR";
+  };
+  vsMode: {
+    id: string;
+    mode: "REGULAR" | "BANKARA" | "PRIVATE" | "FEST";
+  };
+  vsStage: {
+    id: string;
+    name: string;
+    image: Image;
+  };
+  playedTime: string; // 2021-01-01T00:00:00Z
+};
+
+export type BattleExporter<D> = {
+  name: string;
+  getLatestBattleTime: () => Promise<Date>;
+  exportBattle: (detail: D) => Promise<void>;
+};
+
 export type RespMap = {
   [Queries.HomeQuery]: {
     currentPlayer: {
@@ -76,7 +101,9 @@ export type RespMap = {
       historyGroups: HistoryGroups;
     };
   };
-  [Queries.VsHistoryDetailQuery]: Record<never, never>;
+  [Queries.VsHistoryDetailQuery]: {
+    vsHistoryDetail: VsHistoryDetail;
+  };
   [Queries.CoopHistoryQuery]: Record<never, never>;
   [Queries.CoopHistoryDetailQuery]: Record<never, never>;
 };

utils.ts

@@ -23,6 +23,7 @@ export async function readline() {
       return line;
     }
   }
+  throw new Error("EOF");
 }
 
 type PromiseReturnType<T> = T extends () => Promise<infer R> ? R : never;