add stage exporting

deps.ts

@@ -14,3 +14,4 @@ export { MultiProgressBar } from "https://deno.land/x/progress@v1.2.8/mod.ts";
 export { Mutex } from "https://deno.land/x/semaphore@v1.1.1/mod.ts";
 export type { DeepReadonly } from "https://deno.land/x/ts_essentials@v9.1.2/mod.ts";
 export * as MongoDB from "npm:mongodb";
+export * as splatNet3Types from "npm:splatnet3-types/splatnet3";

s3si.ts

@@ -14,6 +14,7 @@ const parseArgs = (args: string[]) => {
       "monitor": ["m"],
       "skipMode": ["s", "skip-mode"],
       "withSummary": "with-summary",
+      "withStages": "with-stages",
     },
   });
   return parsed;
@@ -34,6 +35,7 @@ Options:
     --skip-mode <mode>, -s       Skip mode (default: null)
                                  ("vs", "coop")
     --with-summary               Include summary in the output
+    --with-stages                Include stage records in the output
     --help                       Show this help message and exit`,
   );
   Deno.exit(0);

scripts/convert-old-mongodb-battles.ts

@@ -3,7 +3,7 @@ import { DEFAULT_ENV } from "../src/env.ts";
 import { MongoDBExporter } from "../src/exporters/mongodb.ts";
 import { FileStateBackend, Profile } from "../src/state.ts";
 
-const OLD_BATTLES_END_DATE = new Date('2023-02-28T03:42:47.000+00:00');
+const OLD_BATTLES_END_DATE = new Date("2023-02-28T03:42:47.000+00:00");
 
 const env = DEFAULT_ENV;
 const stateBackend = new FileStateBackend("./profile.json");
@@ -11,18 +11,18 @@ const profile = new Profile({ stateBackend, env });
 await profile.readState();
 
 if (!profile.state.mongoDbUri) {
-	console.error("MongoDB URI not set");
+  console.error("MongoDB URI not set");
-	Deno.exit(1);
+  Deno.exit(1);
 }
 
 const mongoDbClient = new MongoDB.MongoClient(profile.state.mongoDbUri);
 const battlesCollection = mongoDbClient.db("splashcat").collection("battles");
 
 const filter = {
-	"exportDate": {
+  "exportDate": {
-		"$lte": OLD_BATTLES_END_DATE,
+    "$lte": OLD_BATTLES_END_DATE,
-	},
+  },
-	"gameId": undefined,
+  "gameId": undefined,
 };
 
 const cursor = battlesCollection.find(filter);
@@ -32,16 +32,18 @@ const oldDocuments = await battlesCollection.countDocuments(filter);
 console.log(`Found ${oldDocuments} old battles to update...`);
 
 for await (const doc of cursor) {
-	const { splatNetData, _id } = doc;
+  const { splatNetData, _id } = doc;
 
-	const splatNetId = splatNetData.id;
+  const splatNetId = splatNetData.id;
-	const uniqueId = MongoDBExporter.getGameId(splatNetId);
+  const uniqueId = MongoDBExporter.getGameId(splatNetId);
 
-	await battlesCollection.updateOne({ _id }, { "$set": {
+  await battlesCollection.updateOne({ _id }, {
-		gameId: uniqueId,
+    "$set": {
-	}});
+      gameId: uniqueId,
+    },
+  });
 
-	console.log(`Updated ${splatNetId} to ${uniqueId}`);
+  console.log(`Updated ${splatNetId} to ${uniqueId}`);
 }
 
-console.log("Done!")
+console.log("Done!");

scripts/fix-stored-dates.ts

@@ -9,17 +9,17 @@ const profile = new Profile({ stateBackend, env });
 await profile.readState();
 
 if (!profile.state.mongoDbUri) {
-	console.error("MongoDB URI not set");
+  console.error("MongoDB URI not set");
-	Deno.exit(1);
+  Deno.exit(1);
 }
 
 const mongoDbClient = new MongoDB.MongoClient(profile.state.mongoDbUri);
 const battlesCollection = mongoDbClient.db("splashcat").collection("battles");
 
 const filter = {
-	"splatNetData.playedTime": {
+  "splatNetData.playedTime": {
-		$type: "string"
+    $type: "string",
-	}
+  },
 };
 
 const cursor = battlesCollection.find(filter);
@@ -29,13 +29,19 @@ const oldDocuments = await battlesCollection.countDocuments(filter);
 console.log(`Found ${oldDocuments} old battles to update...`);
 
 for await (const doc of cursor) {
-	const { splatNetData, _id } = doc;
+  const { splatNetData, _id } = doc;
 
-	await battlesCollection.updateOne({ _id }, { "$set": {
+  await battlesCollection.updateOne({ _id }, {
-		"splatNetData.playedTime": new Date(splatNetData.playedTime),
+    "$set": {
-	}});
+      "splatNetData.playedTime": new Date(splatNetData.playedTime),
+    },
+  });
 
-	console.log(`Updated ${splatNetData.playedTime} to ${new Date(splatNetData.playedTime)}`);
+  console.log(
+    `Updated ${splatNetData.playedTime} to ${new Date(
+      splatNetData.playedTime,
+    )}`,
+  );
 }
 
-console.log("Done!")
+console.log("Done!");

src/app.ts

@@ -17,6 +17,7 @@ export type Opts = {
   noProgress: boolean;
   monitor: boolean;
   withSummary: boolean;
+  withStages: boolean;
   skipMode?: string;
   cache?: Cache;
   stateBackend?: StateBackend;
@@ -29,6 +30,7 @@ export const DEFAULT_OPTS: Opts = {
   noProgress: false,
   monitor: false,
   withSummary: false,
+  withStages: true,
   env: DEFAULT_ENV,
 };
 
@@ -310,6 +312,34 @@ export class App {
         throw errors[0];
       }
     }
+
+    const stageExporters = exporters.filter((e) => e.exportStages);
+    if (!this.opts.withStages || stageExporters.length === 0) {
+      this.env.logger.log("Skip exporting stages.");
+    } else {
+      const stageRecords = await splatnet.getStageRecords();
+
+      await Promise.all(
+        stageExporters.map((e) =>
+          showError(
+            this.env,
+            e.exportStages!(stageRecords.stageRecords.nodes),
+          ).then((result) => {
+            if (result.status === "success") {
+              this.env.logger.log(`Exported stages to ${result.url}`);
+            } else if (result.status === "skip") {
+              this.env.logger.log(`Skipped exporting stages to ${e.name}`);
+            } else {
+              const _never: never = result;
+            }
+          })
+            .catch((err) => {
+              errors.push(err);
+              this.env.logger.error(`\nFailed to export to ${e.name}:`, err);
+            })
+        ),
+      );
+    }
   }
   async monitor() {
     while (true) {

src/exporters/mongodb.ts

@@ -1,101 +1,113 @@
 import { MongoDB } from "../../deps.ts";
 import { AGENT_VERSION, NSOAPP_VERSION, S3SI_VERSION } from "../constant.ts";
-import { CoopHistoryDetail, ExportResult, Game, GameExporter, Summary, VsHistoryDetail } from "../types.ts";
+import {
+  CoopHistoryDetail,
+  ExportResult,
+  Game,
+  GameExporter,
+  Summary,
+  VsHistoryDetail,
+} from "../types.ts";
 import { parseHistoryDetailId } from "../utils.ts";
 
 export class MongoDBExporter implements GameExporter {
-	name = "mongodb";
+  name = "mongodb";
-	mongoDbClient: MongoDB.MongoClient;
+  mongoDbClient: MongoDB.MongoClient;
-	mongoDb: MongoDB.Db;
+  mongoDb: MongoDB.Db;
-	battlesCollection: MongoDB.Collection;
+  battlesCollection: MongoDB.Collection;
-	jobsCollection: MongoDB.Collection;
+  jobsCollection: MongoDB.Collection;
-	summariesCollection: MongoDB.Collection;
+  summariesCollection: MongoDB.Collection;
-	constructor(private mongoDbUri: string) {
+  constructor(private mongoDbUri: string) {
-		this.mongoDbClient = new MongoDB.MongoClient(mongoDbUri);
+    this.mongoDbClient = new MongoDB.MongoClient(mongoDbUri);
-		this.mongoDb = this.mongoDbClient.db("splashcat");
+    this.mongoDb = this.mongoDbClient.db("splashcat");
-		this.battlesCollection = this.mongoDb.collection("battles");
+    this.battlesCollection = this.mongoDb.collection("battles");
-		this.jobsCollection = this.mongoDb.collection("jobs");
+    this.jobsCollection = this.mongoDb.collection("jobs");
-		this.summariesCollection = this.mongoDb.collection("summaries");
+    this.summariesCollection = this.mongoDb.collection("summaries");
-	}
+  }
 
-	static getGameId(id: string) { // very similar to the file exporter
+  static getGameId(id: string) { // very similar to the file exporter
-		const { uid, timestamp } = parseHistoryDetailId(id);
+    const { uid, timestamp } = parseHistoryDetailId(id);
 
-		return `${uid}_${timestamp}Z`;
+    return `${uid}_${timestamp}Z`;
-	}
+  }
 
-	async notExported({ type, list }: { type: Game["type"], list: string[] }): Promise<string[]> {
+  async notExported(
-		const out: string[] = [];
+    { type, list }: { type: Game["type"]; list: string[] },
+  ): Promise<string[]> {
+    const out: string[] = [];
 
-		const collection = type === "CoopInfo" ? this.jobsCollection : this.battlesCollection;
+    const collection = type === "CoopInfo"
+      ? this.jobsCollection
+      : this.battlesCollection;
 
-		for (const id of list) {
+    for (const id of list) {
-			const uniqueId = MongoDBExporter.getGameId(id);
+      const uniqueId = MongoDBExporter.getGameId(id);
-			const countNewStorage = await collection.countDocuments({
+      const countNewStorage = await collection.countDocuments({
-				gameId: uniqueId,
+        gameId: uniqueId,
-			});
+      });
 
-			if (countNewStorage === 0) {
+      if (countNewStorage === 0) {
-				out.push(id);
+        out.push(id);
-			}
+      }
-		}
+    }
 
-		return out;
+    return out;
-	}
+  }
 
-	async exportGame(game: Game): Promise<ExportResult> {
+  async exportGame(game: Game): Promise<ExportResult> {
-		const uniqueId = MongoDBExporter.getGameId(game.detail.id);
+    const uniqueId = MongoDBExporter.getGameId(game.detail.id);
 
-		const common = {
+    const common = {
-			// this seems like useful data to store...
+      // this seems like useful data to store...
-			// loosely modeled after FileExporterTypeCommon
+      // loosely modeled after FileExporterTypeCommon
-			nsoVersion: NSOAPP_VERSION,
+      nsoVersion: NSOAPP_VERSION,
-			agentVersion: AGENT_VERSION,
+      agentVersion: AGENT_VERSION,
-			s3siVersion: S3SI_VERSION,
+      s3siVersion: S3SI_VERSION,
-			exportDate: new Date(),
+      exportDate: new Date(),
-		};
+    };
 
-		const splatNetData = {
+    const splatNetData = {
-			...game.detail,
+      ...game.detail,
-			playedTime: new Date(game.detail.playedTime),
+      playedTime: new Date(game.detail.playedTime),
-		};
+    };
 
-		const body:
+    const body: {
-		{
+      data: Game;
-			data: Game,
+      splatNetData:
-			splatNetData: Omit<(VsHistoryDetail | CoopHistoryDetail), "playedTime"> & { playedTime: Date },
+        & Omit<(VsHistoryDetail | CoopHistoryDetail), "playedTime">
-			gameId: string,
+        & { playedTime: Date };
-		} & typeof common = {
+      gameId: string;
-			...common,
+    } & typeof common = {
-			data: game,
+      ...common,
-			splatNetData,
+      data: game,
-			gameId: uniqueId,
+      splatNetData,
-		};
+      gameId: uniqueId,
+    };
 
-		const isJob = game.type === "CoopInfo";
+    const isJob = game.type === "CoopInfo";
 
-		const collection = isJob ? this.jobsCollection : this.battlesCollection;
+    const collection = isJob ? this.jobsCollection : this.battlesCollection;
 
-		const result = await collection.insertOne(body);
+    const result = await collection.insertOne(body);
 
-		const objectId = result.insertedId;
+    const objectId = result.insertedId;
 
-		return {
+    return {
-			status: "success",
+      status: "success",
-			url: `https://new.splatoon.catgirlin.space/battle/${objectId.toString()}`,
+      url: `https://new.splatoon.catgirlin.space/battle/${objectId.toString()}`,
-		};
+    };
-	}
+  }
 
-	async exportSummary(summary: Summary): Promise<ExportResult> {
+  async exportSummary(summary: Summary): Promise<ExportResult> {
-		const id = summary.uid;
+    const id = summary.uid;
 
-		await this.summariesCollection.insertOne({
+    await this.summariesCollection.insertOne({
-			summaryId: id,
+      summaryId: id,
-			...summary,
+      ...summary,
-		});
+    });
 
-		return {
+    return {
-			status: "success",
+      status: "success",
-		};
+    };
-	}
+  }
 }

src/splatnet3.ts

@@ -257,6 +257,12 @@ export class Splatnet3 {
       CoopHistoryQuery,
     };
   }
+
+  async getStageRecords() {
+    const resp = await this.request(Queries.StageRecordQuery);
+
+    return resp;
+  }
 }
 
 function getIdsFromGroups<T extends { id: string }>(

src/types.ts

@@ -1,3 +1,4 @@
+import { splatNet3Types } from "../deps.ts";
 import { RankState } from "./state.ts";
 
 export enum Queries {
@@ -15,6 +16,7 @@ export enum Queries {
   myOutfitCommonDataEquipmentsQuery = "d29cd0c2b5e6bac90dd5b817914832f8",
   HistoryRecordQuery = "32b6771f94083d8f04848109b7300af5",
   ConfigureAnalyticsQuery = "f8ae00773cc412a50dd41a6d9a159ddd",
+  StageRecordQuery = "f08a932d533845dde86e674e03bbb7d3",
 }
 export type VarsMap = {
   [Queries.HomeQuery]: [];
@@ -34,6 +36,7 @@ export type VarsMap = {
   [Queries.myOutfitCommonDataEquipmentsQuery]: [];
   [Queries.HistoryRecordQuery]: [];
   [Queries.ConfigureAnalyticsQuery]: [];
+  [Queries.StageRecordQuery]: [];
 };
 
 export type Image = {
@@ -371,6 +374,9 @@ export type GameExporter = {
   ) => Promise<string[]>;
   exportGame: (game: Game) => Promise<ExportResult>;
   exportSummary?: (summary: Summary) => Promise<ExportResult>;
+  exportStages?: (
+    stages: RespMap[Queries.StageRecordQuery]["stageRecords"]["nodes"],
+  ) => Promise<ExportResult>;
 };
 
 export type BankaraBattleHistories = {
@@ -550,6 +556,7 @@ export type RespMap = {
       xMatchMaxLf: SimpleXRank;
     } | null;
   };
+  [Queries.StageRecordQuery]: splatNet3Types.StageRecordResult;
 };
 export type WeaponWithRatio = {
   weapon: {