feat: add upload from file (#30)

* feat: add upload from file

* fix: url is undefined

* feat: support pathname

* feat: read from cache if errored

* fix: state not initialized

* fix: splatnet access

scripts/export-file-to-stat-ink.ts

@@ -0,0 +1,151 @@
+/**
+ * Upload file exporter battles to stat.ink.
+ * Make sure you have already logged in.
+ */
+import { flags } from "../deps.ts";
+import { FileCache } from "../src/cache.ts";
+import { DEFAULT_ENV } from "../src/env.ts";
+import { FileExporter } from "../src/exporters/file.ts";
+import { StatInkExporter } from "../src/exporters/stat.ink.ts";
+import { GameFetcher } from "../src/GameFetcher.ts";
+import { loginManually } from "../src/iksm.ts";
+import { Splatnet3 } from "../src/splatnet3.ts";
+import { FileStateBackend, Profile } from "../src/state.ts";
+import { Game } from "../src/types.ts";
+import { parseHistoryDetailId } from "../src/utils.ts";
+
+async function exportType(
+  { statInkExporter, fileExporter, type, gameFetcher }: {
+    statInkExporter: StatInkExporter;
+    fileExporter: FileExporter;
+    gameFetcher: GameFetcher;
+    type: Game["type"];
+  },
+) {
+  const gameList = await fileExporter.exportedGames({ uid, type });
+
+  const workQueue = [
+    ...await statInkExporter.notExported({
+      type,
+      list: gameList.map((i) => i.id),
+    }),
+  ]
+    .reverse().map((id) => gameList.find((i) => i.id === id)!);
+
+  console.log(`Exporting ${workQueue.length} ${type} games`);
+
+  let exported = 0;
+  for (const { getContent } of workQueue) {
+    const detail = await getContent();
+    let resultUrl: string | undefined;
+    try {
+      const { url } = await statInkExporter.exportGame(detail);
+      resultUrl = url;
+    } catch (e) {
+      console.log("Failed to export game", e);
+      // try to re-export using cached data
+      const cachedDetail =
+        (await gameFetcher.fetch(type, detail.detail.id)).detail;
+      const { detail: _, ...rest } = detail;
+      // @ts-ignore the type must be the same
+      const { url } = await statInkExporter.exportGame({
+        ...rest,
+        detail: cachedDetail,
+      });
+      resultUrl = url;
+    }
+    exported += 1;
+    if (resultUrl) {
+      console.log(`Exported ${resultUrl} (${exported}/${workQueue.length})`);
+    }
+  }
+}
+
+const parseArgs = (args: string[]) => {
+  const parsed = flags.parse(args, {
+    string: ["profilePath", "type"],
+    alias: {
+      "help": "h",
+      "profilePath": ["p", "profile-path"],
+    },
+  });
+
+  return parsed;
+};
+
+const opts = parseArgs(Deno.args);
+if (opts.help) {
+  console.log(
+    `Usage: deno run -A ${Deno.mainModule} [options]
+  
+  Options:
+      --type                       Type of game to export. Can be vs, coop, or all. (default: coop)
+      --profile-path <path>, -p    Path to config file (default: ./profile.json)
+      --help                       Show this help message and exit`,
+  );
+  Deno.exit(0);
+}
+
+const env = DEFAULT_ENV;
+const stateBackend = new FileStateBackend(opts.profilePath ?? "./profile.json");
+const profile = new Profile({ stateBackend, env });
+await profile.readState();
+
+// for cache
+const gameFetcher = new GameFetcher({
+  cache: new FileCache(profile.state.cacheDir),
+  state: profile.state,
+});
+
+if (!profile.state.loginState?.sessionToken) {
+  const sessionToken = await loginManually(env);
+
+  await profile.writeState({
+    ...profile.state,
+    loginState: {
+      ...profile.state.loginState,
+      sessionToken,
+    },
+  });
+}
+
+const splatnet = new Splatnet3({ profile, env });
+
+console.log("Fetching uid...");
+const { latestBattleHistories: { historyGroups } } = await splatnet
+  .getLatestBattleHistoriesQuery();
+
+const id = historyGroups.nodes?.[0].historyDetails.nodes?.[0].id;
+
+if (!id) {
+  console.log("No battle history found");
+  Deno.exit(0);
+}
+
+const { uid } = parseHistoryDetailId(id);
+
+const fileExporter = new FileExporter(profile.state.fileExportPath);
+const statInkExporter = new StatInkExporter({
+  statInkApiKey: profile.state.statInkApiKey!,
+  uploadMode: "Manual",
+  env,
+});
+const type = (opts.type ?? "coop").replace("all", "vs,coop");
+
+if (type.includes("vs")) {
+  await exportType({
+    type: "VsInfo",
+    fileExporter,
+    statInkExporter,
+    gameFetcher,
+  });
+}
+
+if (type.includes("coop")) {
+  await exportType({
+    type: "CoopInfo",
+    fileExporter,
+    statInkExporter,
+    gameFetcher,
+  });
+}

src/GameFetcher.ts

@@ -16,29 +16,36 @@ import { RankTracker } from "./RankTracker.ts";
 
 /**
  * Fetch game and cache it. It also fetches bankara match challenge info.
+ * if splatnet is not given, it will use cache only
  */
 export class GameFetcher {
-  splatnet: Splatnet3;
+  private _splatnet?: Splatnet3;
-  cache: Cache;
+  private cache: Cache;
-  rankTracker: RankTracker;
+  private rankTracker: RankTracker;
 
-  lock: Record<string, Mutex | undefined> = {};
+  private lock: Record<string, Mutex | undefined> = {};
-  bankaraLock = new Mutex();
+  private bankaraLock = new Mutex();
-  bankaraHistory?: HistoryGroups<BattleListNode>["nodes"];
+  private bankaraHistory?: HistoryGroups<BattleListNode>["nodes"];
-  coopLock = new Mutex();
+  private coopLock = new Mutex();
-  coopHistory?: CoopHistoryGroups["nodes"];
+  private coopHistory?: CoopHistoryGroups["nodes"];
 
   constructor(
     { cache = new MemoryCache(), splatnet, state }: {
-      splatnet: Splatnet3;
+      splatnet?: Splatnet3;
       state: State;
       cache?: Cache;
     },
   ) {
-    this.splatnet = splatnet;
+    this._splatnet = splatnet;
     this.cache = cache;
     this.rankTracker = new RankTracker(state.rankState);
   }
+  private get splatnet() {
+    if (!this._splatnet) {
+      throw new Error("splatnet is not set");
+    }
+    return this._splatnet;
+  }
   private getLock(id: string): Mutex {
     let cur = this.lock[id];
 
@@ -94,7 +101,7 @@ export class GameFetcher {
     });
   }
   async getCoopMetaById(id: string): Promise<Omit<CoopInfo, "detail">> {
-    const coopHistory = await this.getCoopHistory();
+    const coopHistory = this._splatnet ? await this.getCoopHistory() : [];
     const group = coopHistory.find((i) =>
       i.historyDetails.nodes.some((i) => i.id === id)
     );
@@ -119,7 +126,7 @@ export class GameFetcher {
   }
   async getBattleMetaById(id: string): Promise<Omit<VsInfo, "detail">> {
     const gid = await gameId(id);
-    const bankaraHistory = await this.getBankaraHistory();
+    const bankaraHistory = this._splatnet ? await this.getBankaraHistory() : [];
     const gameIdMap = new Map<BattleListNode, string>();
 
     for (const i of bankaraHistory) {
@@ -175,7 +182,7 @@ export class GameFetcher {
       rankBeforeState: before ?? null,
     };
   }
-  cacheDetail<T>(
+  private cacheDetail<T>(
     id: string,
     getter: () => Promise<T>,
   ): Promise<T> {
@@ -204,7 +211,7 @@ export class GameFetcher {
         throw new Error(`Unknown game type: ${type}`);
     }
   }
-  async fetchBattle(id: string): Promise<VsInfo> {
+  private async fetchBattle(id: string): Promise<VsInfo> {
     const detail = await this.cacheDetail(
       id,
       () => this.splatnet.getBattleDetail(id).then((r) => r.vsHistoryDetail),
@@ -218,7 +225,7 @@ export class GameFetcher {
 
     return game;
   }
-  async fetchCoop(id: string): Promise<CoopInfo> {
+  private async fetchCoop(id: string): Promise<CoopInfo> {
     const detail = await this.cacheDetail(
       id,
       () => this.splatnet.getCoopDetail(id).then((r) => r.coopHistoryDetail),

src/exporters/file.ts

@@ -1,4 +1,4 @@
-import { CoopInfo, GameExporter, VsInfo } from "../types.ts";
+import { CoopInfo, Game, GameExporter, VsInfo } from "../types.ts";
 import { path } from "../../deps.ts";
 import { NSOAPP_VERSION, S3SI_VERSION } from "../constant.ts";
 import { parseHistoryDetailId, urlSimplify } from "../utils.ts";
@@ -37,7 +37,53 @@ export class FileExporter implements GameExporter {
 
     return `${uid}_${timestamp}Z.json`;
   }
-  async exportGame(info: VsInfo | CoopInfo) {
+  /**
+   * Get all exported files
+   */
+  async exportedGames(
+    { uid, type }: { uid: string; type: Game["type"] },
+  ): Promise<{ id: string; getContent: () => Promise<Game> }[]> {
+    const out: { id: string; filepath: string; timestamp: string }[] = [];
+
+    for await (const entry of Deno.readDir(this.exportPath)) {
+      const filename = entry.name;
+      const [fileUid, timestamp] = filename.split("_", 2);
+      if (!entry.isFile || fileUid !== uid) {
+        continue;
+      }
+
+      const filepath = path.join(this.exportPath, filename);
+      const content = await Deno.readTextFile(filepath);
+      const body = JSON.parse(content) as FileExporterType;
+
+      if (body.type === "VS" && type === "VsInfo") {
+        out.push({
+          id: body.data.detail.id,
+          filepath,
+          timestamp,
+        });
+      } else if (body.type === "COOP" && type === "CoopInfo") {
+        out.push({
+          id: body.data.detail.id,
+          filepath,
+          timestamp,
+        });
+      }
+    }
+
+    return out.sort((a, b) => b.timestamp.localeCompare(a.timestamp)).map((
+      { id, filepath },
+    ) => ({
+      id,
+      getContent: async () => {
+        const content = await Deno.readTextFile(filepath);
+        const body = JSON.parse(content) as FileExporterType;
+
+        return body.data;
+      },
+    }));
+  }
+  async exportGame(info: Game) {
     await Deno.mkdir(this.exportPath, { recursive: true });
 
     const filename = this.getFilenameById(info.detail.id);

src/exporters/stat.ink.ts

@@ -29,7 +29,13 @@ import {
 } from "../types.ts";
 import { msgpack, Mutex } from "../../deps.ts";
 import { APIError } from "../APIError.ts";
-import { b64Number, gameId, nonNullable, s3siGameId } from "../utils.ts";
+import {
+  b64Number,
+  gameId,
+  nonNullable,
+  s3siGameId,
+  urlSimplify,
+} from "../utils.ts";
 import { Env } from "../env.ts";
 import { KEY_DICT } from "../dict/stat.ink.ts";
 
@@ -458,9 +464,17 @@ export class StatInkExporter implements GameExporter {
     return result;
   }
   isRandomWeapon(image: Image | null): boolean {
-    return (image?.url.includes(
+    const RANDOM_WEAPON_FILENAME =
-      "473fffb2442075078d8bb7125744905abdeae651b6a5b7453ae295582e45f7d1",
+      "473fffb2442075078d8bb7125744905abdeae651b6a5b7453ae295582e45f7d1";
-    )) ?? false;
+    // file exporter will replace url to { pathname: string } | string
+    const url = image?.url as ReturnType<typeof urlSimplify> | undefined | null;
+    if (typeof url === "string") {
+      return url.includes(RANDOM_WEAPON_FILENAME);
+    } else if (url === undefined || url === null) {
+      return false;
+    } else {
+      return url.pathname.includes(RANDOM_WEAPON_FILENAME);
+    }
   }
   async mapCoopWeapon(
     { name, image }: { name: string; image: Image | null },

src/types.ts

@@ -31,7 +31,7 @@ export type VarsMap = {
 };
 
 export type Image = {
-  url: string;
+  url?: string;
   width?: number;
   height?: number;
 };