add stage record query

mongodb exporter can now identify what already exists and is compatible with the previous format of splashcat's db.

deno.lock

@@ -68,5 +68,117 @@
     "https://deno.land/x/ts_essentials@v9.1.2/lib/mod.ts": "d7e44a25aa621425ffd118a0210a492c5c354411018e2db648a68614d5901f5b",
     "https://deno.land/x/ts_essentials@v9.1.2/lib/types.ts": "7ee99797a880948c07020e90d569ca3c5d465c378949262110283aa7856f5603",
     "https://deno.land/x/ts_essentials@v9.1.2/mod.ts": "ffae461c16d4a1bf24c2179582ab8d5c81ad0df61e4ae2fba51ef5e5bdf90345"
+  },
+  "npm": {
+    "specifiers": {
+      "mongodb": "mongodb@5.5.0",
+      "splatnet3-types": "splatnet3-types@0.2.20230227204004"
+    },
+    "packages": {
+      "@types/node@18.14.2": {
+        "integrity": "sha512-1uEQxww3DaghA0RxqHx0O0ppVlo43pJhepY51OxuQIKHpjbnYLA7vcdwioNPzIqmC2u3I/dmylcqjlh0e7AyUA==",
+        "dependencies": {}
+      },
+      "@types/webidl-conversions@7.0.0": {
+        "integrity": "sha512-xTE1E+YF4aWPJJeUzaZI5DRntlkY3+BCVJi0axFptnjGmAoWxkyREIh/XMrfxVLejwQxMCfDXdICo0VLxThrog==",
+        "dependencies": {}
+      },
+      "@types/whatwg-url@8.2.2": {
+        "integrity": "sha512-FtQu10RWgn3D9U4aazdwIE2yzphmTJREDqNdODHrbrZmmMqI0vMheC/6NE/J1Yveaj8H+ela+YwWTjq5PGmuhA==",
+        "dependencies": {
+          "@types/node": "@types/node@18.14.2",
+          "@types/webidl-conversions": "@types/webidl-conversions@7.0.0"
+        }
+      },
+      "bson@5.0.1": {
+        "integrity": "sha512-y09gBGusgHtinMon/GVbv1J6FrXhnr/+6hqLlSmEFzkz6PodqF6TxjyvfvY3AfO+oG1mgUtbC86xSbOlwvM62Q==",
+        "dependencies": {}
+      },
+      "bson@5.3.0": {
+        "integrity": "sha512-ukmCZMneMlaC5ebPHXIkP8YJzNl5DC41N5MAIvKDqLggdao342t4McltoJBQfQya/nHBWAcSsYRqlXPoQkTJag==",
+        "dependencies": {}
+      },
+      "ip@2.0.0": {
+        "integrity": "sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==",
+        "dependencies": {}
+      },
+      "memory-pager@1.5.0": {
+        "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==",
+        "dependencies": {}
+      },
+      "mongodb-connection-string-url@2.6.0": {
+        "integrity": "sha512-WvTZlI9ab0QYtTYnuMLgobULWhokRjtC7db9LtcVfJ+Hsnyr5eo6ZtNAt3Ly24XZScGMelOcGtm7lSn0332tPQ==",
+        "dependencies": {
+          "@types/whatwg-url": "@types/whatwg-url@8.2.2",
+          "whatwg-url": "whatwg-url@11.0.0"
+        }
+      },
+      "mongodb@5.1.0": {
+        "integrity": "sha512-qgKb7y+EI90y4weY3z5+lIgm8wmexbonz0GalHkSElQXVKtRuwqXuhXKccyvIjXCJVy9qPV82zsinY0W1FBnJw==",
+        "dependencies": {
+          "bson": "bson@5.0.1",
+          "mongodb-connection-string-url": "mongodb-connection-string-url@2.6.0",
+          "saslprep": "saslprep@1.0.3",
+          "socks": "socks@2.7.1"
+        }
+      },
+      "mongodb@5.5.0": {
+        "integrity": "sha512-XgrkUgAAdfnZKQfk5AsYL8j7O99WHd4YXPxYxnh8dZxD+ekYWFRA3JktUsBnfg+455Smf75/+asoU/YLwNGoQQ==",
+        "dependencies": {
+          "bson": "bson@5.3.0",
+          "mongodb-connection-string-url": "mongodb-connection-string-url@2.6.0",
+          "saslprep": "saslprep@1.0.3",
+          "socks": "socks@2.7.1"
+        }
+      },
+      "punycode@2.3.0": {
+        "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==",
+        "dependencies": {}
+      },
+      "saslprep@1.0.3": {
+        "integrity": "sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag==",
+        "dependencies": {
+          "sparse-bitfield": "sparse-bitfield@3.0.3"
+        }
+      },
+      "smart-buffer@4.2.0": {
+        "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==",
+        "dependencies": {}
+      },
+      "socks@2.7.1": {
+        "integrity": "sha512-7maUZy1N7uo6+WVEX6psASxtNlKaNVMlGQKkG/63nEDdLOWNbiUMoLK7X4uYoLhQstau72mLgfEWcXcwsaHbYQ==",
+        "dependencies": {
+          "ip": "ip@2.0.0",
+          "smart-buffer": "smart-buffer@4.2.0"
+        }
+      },
+      "sparse-bitfield@3.0.3": {
+        "integrity": "sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==",
+        "dependencies": {
+          "memory-pager": "memory-pager@1.5.0"
+        }
+      },
+      "splatnet3-types@0.2.20230227204004": {
+        "integrity": "sha512-FAY6pbUcrp5O8c49BNXSKxoyM3UlCrRx2AtA9Y3qlvqOLdHqwxtzcdzbk1b1hRam8ZcrxRzE/ii6ESRiPIAnZw==",
+        "dependencies": {}
+      },
+      "tr46@3.0.0": {
+        "integrity": "sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==",
+        "dependencies": {
+          "punycode": "punycode@2.3.0"
+        }
+      },
+      "webidl-conversions@7.0.0": {
+        "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==",
+        "dependencies": {}
+      },
+      "whatwg-url@11.0.0": {
+        "integrity": "sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==",
+        "dependencies": {
+          "tr46": "tr46@3.0.0",
+          "webidl-conversions": "webidl-conversions@7.0.0"
+        }
+      }
+    }
   }
 }

deps.ts

@@ -13,4 +13,6 @@ export * as path from "https://deno.land/std@0.160.0/path/mod.ts";
 export { MultiProgressBar } from "https://deno.land/x/progress@v1.2.8/mod.ts";
 export { Mutex } from "https://deno.land/x/semaphore@v1.1.1/mod.ts";
 export type { DeepReadonly } from "https://deno.land/x/ts_essentials@v9.1.2/mod.ts";
+export * as MongoDB from "npm:mongodb";
+export * as splatNet3Types from "npm:splatnet3-types/splatnet3";
 export { writeAll } from "https://deno.land/std@0.160.0/streams/conversion.ts";

s3si.ts

@@ -14,6 +14,7 @@ const parseArgs = (args: string[]) => {
       "monitor": ["m"],
       "skipMode": ["s", "skip-mode"],
       "withSummary": "with-summary",
+      "withStages": "with-stages",
     },
   });
   return parsed;
@@ -28,12 +29,13 @@ Options:
     --profile-path <path>, -p    Path to config file (default: ./profile.json)
     --exporter <exporter>, -e    Exporter list to use (default: stat.ink)
                                  Multiple exporters can be separated by commas
-                                 (e.g. "stat.ink,file")
+                                 (e.g. "stat.ink,file,mongodb")
     --no-progress, -n            Disable progress bar
     --monitor, -m                Monitor mode
     --skip-mode <mode>, -s       Skip mode (default: null)
                                  ("vs", "coop")
     --with-summary               Include summary in the output
+    --with-stages                Include stage records in the output
     --help                       Show this help message and exit`,
   );
   Deno.exit(0);

scripts/convert-old-mongodb-battles.ts

@@ -0,0 +1,49 @@
+import { MongoDB } from "../deps.ts";
+import { DEFAULT_ENV } from "../src/env.ts";
+import { MongoDBExporter } from "../src/exporters/mongodb.ts";
+import { FileStateBackend, Profile } from "../src/state.ts";
+
+const OLD_BATTLES_END_DATE = new Date("2023-02-28T03:42:47.000+00:00");
+
+const env = DEFAULT_ENV;
+const stateBackend = new FileStateBackend("./profile.json");
+const profile = new Profile({ stateBackend, env });
+await profile.readState();
+
+if (!profile.state.mongoDbUri) {
+  console.error("MongoDB URI not set");
+  Deno.exit(1);
+}
+
+const mongoDbClient = new MongoDB.MongoClient(profile.state.mongoDbUri);
+const battlesCollection = mongoDbClient.db("splashcat").collection("battles");
+
+const filter = {
+  "exportDate": {
+    "$lte": OLD_BATTLES_END_DATE,
+  },
+  "gameId": undefined,
+};
+
+const cursor = battlesCollection.find(filter);
+
+const oldDocuments = await battlesCollection.countDocuments(filter);
+
+console.log(`Found ${oldDocuments} old battles to update...`);
+
+for await (const doc of cursor) {
+  const { splatNetData, _id } = doc;
+
+  const splatNetId = splatNetData.id;
+  const uniqueId = MongoDBExporter.getGameId(splatNetId);
+
+  await battlesCollection.updateOne({ _id }, {
+    "$set": {
+      gameId: uniqueId,
+    },
+  });
+
+  console.log(`Updated ${splatNetId} to ${uniqueId}`);
+}
+
+console.log("Done!");

scripts/deno.lock

@@ -84,5 +84,104 @@
     "https://deno.land/x/ts_essentials@v9.1.2/lib/mod.ts": "d7e44a25aa621425ffd118a0210a492c5c354411018e2db648a68614d5901f5b",
     "https://deno.land/x/ts_essentials@v9.1.2/lib/types.ts": "7ee99797a880948c07020e90d569ca3c5d465c378949262110283aa7856f5603",
     "https://deno.land/x/ts_essentials@v9.1.2/mod.ts": "ffae461c16d4a1bf24c2179582ab8d5c81ad0df61e4ae2fba51ef5e5bdf90345"
+  },
+  "npm": {
+    "specifiers": {
+      "mongodb": "mongodb@5.1.0",
+      "splatnet3-types": "splatnet3-types@0.2.20230227204004"
+    },
+    "packages": {
+      "@types/node@18.14.2": {
+        "integrity": "sha512-1uEQxww3DaghA0RxqHx0O0ppVlo43pJhepY51OxuQIKHpjbnYLA7vcdwioNPzIqmC2u3I/dmylcqjlh0e7AyUA==",
+        "dependencies": {}
+      },
+      "@types/webidl-conversions@7.0.0": {
+        "integrity": "sha512-xTE1E+YF4aWPJJeUzaZI5DRntlkY3+BCVJi0axFptnjGmAoWxkyREIh/XMrfxVLejwQxMCfDXdICo0VLxThrog==",
+        "dependencies": {}
+      },
+      "@types/whatwg-url@8.2.2": {
+        "integrity": "sha512-FtQu10RWgn3D9U4aazdwIE2yzphmTJREDqNdODHrbrZmmMqI0vMheC/6NE/J1Yveaj8H+ela+YwWTjq5PGmuhA==",
+        "dependencies": {
+          "@types/node": "@types/node@18.14.2",
+          "@types/webidl-conversions": "@types/webidl-conversions@7.0.0"
+        }
+      },
+      "bson@5.0.1": {
+        "integrity": "sha512-y09gBGusgHtinMon/GVbv1J6FrXhnr/+6hqLlSmEFzkz6PodqF6TxjyvfvY3AfO+oG1mgUtbC86xSbOlwvM62Q==",
+        "dependencies": {}
+      },
+      "ip@2.0.0": {
+        "integrity": "sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==",
+        "dependencies": {}
+      },
+      "memory-pager@1.5.0": {
+        "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==",
+        "dependencies": {}
+      },
+      "mongodb-connection-string-url@2.6.0": {
+        "integrity": "sha512-WvTZlI9ab0QYtTYnuMLgobULWhokRjtC7db9LtcVfJ+Hsnyr5eo6ZtNAt3Ly24XZScGMelOcGtm7lSn0332tPQ==",
+        "dependencies": {
+          "@types/whatwg-url": "@types/whatwg-url@8.2.2",
+          "whatwg-url": "whatwg-url@11.0.0"
+        }
+      },
+      "mongodb@5.1.0": {
+        "integrity": "sha512-qgKb7y+EI90y4weY3z5+lIgm8wmexbonz0GalHkSElQXVKtRuwqXuhXKccyvIjXCJVy9qPV82zsinY0W1FBnJw==",
+        "dependencies": {
+          "bson": "bson@5.0.1",
+          "mongodb-connection-string-url": "mongodb-connection-string-url@2.6.0",
+          "saslprep": "saslprep@1.0.3",
+          "socks": "socks@2.7.1"
+        }
+      },
+      "punycode@2.3.0": {
+        "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==",
+        "dependencies": {}
+      },
+      "saslprep@1.0.3": {
+        "integrity": "sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag==",
+        "dependencies": {
+          "sparse-bitfield": "sparse-bitfield@3.0.3"
+        }
+      },
+      "smart-buffer@4.2.0": {
+        "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==",
+        "dependencies": {}
+      },
+      "socks@2.7.1": {
+        "integrity": "sha512-7maUZy1N7uo6+WVEX6psASxtNlKaNVMlGQKkG/63nEDdLOWNbiUMoLK7X4uYoLhQstau72mLgfEWcXcwsaHbYQ==",
+        "dependencies": {
+          "ip": "ip@2.0.0",
+          "smart-buffer": "smart-buffer@4.2.0"
+        }
+      },
+      "sparse-bitfield@3.0.3": {
+        "integrity": "sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==",
+        "dependencies": {
+          "memory-pager": "memory-pager@1.5.0"
+        }
+      },
+      "splatnet3-types@0.2.20230227204004": {
+        "integrity": "sha512-FAY6pbUcrp5O8c49BNXSKxoyM3UlCrRx2AtA9Y3qlvqOLdHqwxtzcdzbk1b1hRam8ZcrxRzE/ii6ESRiPIAnZw==",
+        "dependencies": {}
+      },
+      "tr46@3.0.0": {
+        "integrity": "sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==",
+        "dependencies": {
+          "punycode": "punycode@2.3.0"
+        }
+      },
+      "webidl-conversions@7.0.0": {
+        "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==",
+        "dependencies": {}
+      },
+      "whatwg-url@11.0.0": {
+        "integrity": "sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==",
+        "dependencies": {
+          "tr46": "tr46@3.0.0",
+          "webidl-conversions": "webidl-conversions@7.0.0"
+        }
+      }
+    }
   }
 }

scripts/fix-stored-dates.ts

@@ -0,0 +1,47 @@
+import { MongoDB } from "../deps.ts";
+import { DEFAULT_ENV } from "../src/env.ts";
+import { MongoDBExporter } from "../src/exporters/mongodb.ts";
+import { FileStateBackend, Profile } from "../src/state.ts";
+
+const env = DEFAULT_ENV;
+const stateBackend = new FileStateBackend("./profile.json");
+const profile = new Profile({ stateBackend, env });
+await profile.readState();
+
+if (!profile.state.mongoDbUri) {
+  console.error("MongoDB URI not set");
+  Deno.exit(1);
+}
+
+const mongoDbClient = new MongoDB.MongoClient(profile.state.mongoDbUri);
+const battlesCollection = mongoDbClient.db("splashcat").collection("battles");
+
+const filter = {
+  "splatNetData.playedTime": {
+    $type: "string",
+  },
+};
+
+const cursor = battlesCollection.find(filter);
+
+const oldDocuments = await battlesCollection.countDocuments(filter);
+
+console.log(`Found ${oldDocuments} old battles to update...`);
+
+for await (const doc of cursor) {
+  const { splatNetData, _id } = doc;
+
+  await battlesCollection.updateOne({ _id }, {
+    "$set": {
+      "splatNetData.playedTime": new Date(splatNetData.playedTime),
+    },
+  });
+
+  console.log(
+    `Updated ${splatNetData.playedTime} to ${new Date(
+      splatNetData.playedTime,
+    )}`,
+  );
+}
+
+console.log("Done!");

scripts/send-mongodb-battles-to-splashcat.ts

@@ -0,0 +1,77 @@
+import { MongoDB } from "../deps.ts";
+import { DEFAULT_ENV } from "../src/env.ts";
+import { MongoDBExporter } from "../src/exporters/mongodb.ts";
+import { FileStateBackend, Profile } from "../src/state.ts";
+
+const env = DEFAULT_ENV;
+const stateBackend = new FileStateBackend("./profile.json");
+const profile = new Profile({ stateBackend, env });
+await profile.readState();
+
+if (!profile.state.mongoDbUri) {
+  console.error("MongoDB URI not set");
+  Deno.exit(1);
+}
+
+const mongoDbClient = new MongoDB.MongoClient(profile.state.mongoDbUri);
+const battlesCollection = mongoDbClient.db("splashcat").collection("battles");
+
+const cursor = battlesCollection.find();
+
+const oldDocuments = await battlesCollection.countDocuments();
+
+console.log(`Found ${oldDocuments} old battles to upload...`);
+
+let count = 0;
+
+const erroredBattles = [];
+
+for await (const doc of cursor) {
+  const { splatNetData, _id } = doc;
+
+  // start time for performance tracking, needs to be very accurate
+  const startTime = new Date();
+
+  splatNetData.playedTime = splatNetData.playedTime.toISOString();
+
+  const response = await fetch("http://127.0.0.1:8000/battles/api/upload/", {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      "Authorization": `Bearer ${profile.state.splashcatApiKey}`,
+    },
+    body: JSON.stringify({
+      "data_type": "splatnet3",
+      "battle": splatNetData,
+    })
+  })
+
+  if (!response.ok) {
+    console.error(`Failed to upload ${splatNetData.id}`);
+    erroredBattles.push({
+      id: doc.gameId,
+      error: await response.text(),
+    });
+  }
+
+  // end time for performance tracking, needs to be very accurate
+  const endTime = new Date();
+  const timeTaken = endTime.getTime() - startTime.getTime();
+  
+  console.log(`Uploaded ${splatNetData.id} (${timeTaken}ms)`);
+  count++;
+  console.log(`Uploaded ${count}/${oldDocuments} battles`)
+
+  if (count % 100 === 0) {
+    console.log("Updating error logs...");
+    if (erroredBattles.length > 0) {
+      await Deno.writeFile("./errored-battles.json", new TextEncoder().encode(JSON.stringify(erroredBattles, null, "\t")));
+    }
+  }
+}
+
+console.log("Done!");
+
+if (erroredBattles.length > 0) {
+  await Deno.writeFile("./errored-battles.json", new TextEncoder().encode(JSON.stringify(erroredBattles, null, 2)));
+}

src/app.ts

@@ -9,6 +9,7 @@ import { FileExporter } from "./exporters/file.ts";
 import { delay, showError } from "./utils.ts";
 import { GameFetcher } from "./GameFetcher.ts";
 import { DEFAULT_ENV, Env } from "./env.ts";
+import { MongoDBExporter } from "./exporters/mongodb.ts";
 
 export type Opts = {
   profilePath: string;
@@ -16,6 +17,7 @@ export type Opts = {
   noProgress: boolean;
   monitor: boolean;
   withSummary: boolean;
+  withStages: boolean;
   skipMode?: string;
   cache?: Cache;
   stateBackend?: StateBackend;
@@ -28,6 +30,7 @@ export const DEFAULT_OPTS: Opts = {
   noProgress: false,
   monitor: false,
   withSummary: false,
+  withStages: true,
   env: DEFAULT_ENV,
 };
 
@@ -115,6 +118,25 @@ export class App {
       out.push(new FileExporter(state.fileExportPath));
     }
 
+    if (exporters.includes("mongodb")) {
+      if (!state.mongoDbUri) {
+        const uri = (await this.env.prompts.prompt(
+          "MongoDB URI is not set. Please enter below.",
+        )).trim();
+        if (!uri) {
+          this.env.logger.error("MongoDB URI is required.");
+          Deno.exit(1);
+        }
+        await this.profile.writeState({
+          ...state,
+          mongoDbUri: uri,
+        });
+      }
+      out.push(
+        new MongoDBExporter(this.profile.state.mongoDbUri!),
+      );
+    }
+
     return out;
   }
   exporterProgress(title: string) {
@@ -290,6 +312,34 @@ export class App {
         throw errors[0];
       }
     }
+
+    const stageExporters = exporters.filter((e) => e.exportStages);
+    if (!this.opts.withStages || stageExporters.length === 0) {
+      this.env.logger.log("Skip exporting stages.");
+    } else {
+      const stageRecords = await splatnet.getStageRecords();
+
+      await Promise.all(
+        stageExporters.map((e) =>
+          showError(
+            this.env,
+            e.exportStages!(stageRecords.stageRecords.nodes),
+          ).then((result) => {
+            if (result.status === "success") {
+              this.env.logger.log(`Exported stages to ${result.url}`);
+            } else if (result.status === "skip") {
+              this.env.logger.log(`Skipped exporting stages to ${e.name}`);
+            } else {
+              const _never: never = result;
+            }
+          })
+            .catch((err) => {
+              errors.push(err);
+              this.env.logger.error(`\nFailed to export to ${e.name}:`, err);
+            })
+        ),
+      );
+    }
   }
   async monitor() {
     while (true) {

src/constant.ts

@@ -1,7 +1,9 @@
 import type { StatInkPostBody, VsHistoryDetail } from "./types.ts";
 
-export const AGENT_NAME = "s3si.ts";
+export const AGENT_NAME = "splashcat / s3si.ts";
+export const AGENT_VERSION = "1.1.1";
 export const S3SI_VERSION = "0.4.1";
+export const COMBINED_VERSION = `${AGENT_VERSION}/${S3SI_VERSION}`;
 export const NSOAPP_VERSION = "2.5.1";
 export const WEB_VIEW_VERSION = "4.0.0-d5178440";
 export enum Queries {
@@ -19,10 +21,11 @@ export enum Queries {
   myOutfitCommonDataEquipmentsQuery = "d29cd0c2b5e6bac90dd5b817914832f8",
   HistoryRecordQuery = "d9246baf077b2a29b5f7aac321810a77",
   ConfigureAnalyticsQuery = "f8ae00773cc412a50dd41a6d9a159ddd",
+  StageRecordQuery = "f08a932d533845dde86e674e03bbb7d3",
 }
-export const S3SI_LINK = "https://github.com/spacemeowx2/s3si.ts";
+export const S3SI_LINK = "https://forgejo.catgirlin.space/catgirl/s3si.ts";
 
-export const USERAGENT = `${AGENT_NAME}/${S3SI_VERSION} (${S3SI_LINK})`;
+export const USERAGENT = `${AGENT_NAME}/(${COMBINED_VERSION}) (${S3SI_LINK})`;
 export const DEFAULT_APP_USER_AGENT =
   "Mozilla/5.0 (Linux; Android 11; Pixel 5) " +
   "AppleWebKit/537.36 (KHTML, like Gecko) " +

src/exporters/mongodb.ts

@@ -0,0 +1,133 @@
+import { MongoDB } from "../../deps.ts";
+import { AGENT_VERSION, NSOAPP_VERSION, S3SI_VERSION } from "../constant.ts";
+import {
+  CoopHistoryDetail,
+  ExportResult,
+  Game,
+  GameExporter,
+  Queries,
+  RespMap,
+  Summary,
+  VsHistoryDetail,
+} from "../types.ts";
+import { parseHistoryDetailId } from "../utils.ts";
+
+export class MongoDBExporter implements GameExporter {
+  name = "mongodb";
+  mongoDbClient: MongoDB.MongoClient;
+  mongoDb: MongoDB.Db;
+  battlesCollection: MongoDB.Collection;
+  jobsCollection: MongoDB.Collection;
+  summariesCollection: MongoDB.Collection;
+  constructor(private mongoDbUri: string) {
+    this.mongoDbClient = new MongoDB.MongoClient(mongoDbUri);
+    this.mongoDb = this.mongoDbClient.db("splashcat");
+    this.battlesCollection = this.mongoDb.collection("battles");
+    this.jobsCollection = this.mongoDb.collection("jobs");
+    this.summariesCollection = this.mongoDb.collection("summaries");
+  }
+
+  static getGameId(id: string) { // very similar to the file exporter
+    const { uid, timestamp } = parseHistoryDetailId(id);
+
+    return `${uid}_${timestamp}Z`;
+  }
+
+  async notExported(
+    { type, list }: { type: Game["type"]; list: string[] },
+  ): Promise<string[]> {
+    const out: string[] = [];
+
+    const collection = type === "CoopInfo"
+      ? this.jobsCollection
+      : this.battlesCollection;
+
+    for (const id of list) {
+      const uniqueId = MongoDBExporter.getGameId(id);
+      const countNewStorage = await collection.countDocuments({
+        gameId: uniqueId,
+      });
+
+      if (countNewStorage === 0) {
+        out.push(id);
+      }
+    }
+
+    return out;
+  }
+
+  async exportGame(game: Game): Promise<ExportResult> {
+    const uniqueId = MongoDBExporter.getGameId(game.detail.id);
+
+    const common = {
+      // this seems like useful data to store...
+      // loosely modeled after FileExporterTypeCommon
+      nsoVersion: NSOAPP_VERSION,
+      agentVersion: AGENT_VERSION,
+      s3siVersion: S3SI_VERSION,
+      exportDate: new Date(),
+    };
+
+    const splatNetData = {
+      ...game.detail,
+      playedTime: new Date(game.detail.playedTime),
+    };
+
+    const body: {
+      data: Game;
+      splatNetData:
+        & Omit<(VsHistoryDetail | CoopHistoryDetail), "playedTime">
+        & { playedTime: Date };
+      gameId: string;
+    } & typeof common = {
+      ...common,
+      data: game,
+      splatNetData,
+      gameId: uniqueId,
+    };
+
+    const isJob = game.type === "CoopInfo";
+
+    const collection = isJob ? this.jobsCollection : this.battlesCollection;
+
+    const result = await collection.insertOne(body);
+
+    const objectId = result.insertedId;
+
+    return {
+      status: "success",
+      url: `https://new.splatoon.catgirlin.space/battle/${objectId.toString()}`,
+    };
+  }
+
+  async exportSummary(summary: Summary): Promise<ExportResult> {
+    const id = summary.uid;
+
+    await this.summariesCollection.insertOne({
+      summaryId: id,
+      ...summary,
+    });
+
+    return {
+      status: "success",
+    };
+  }
+
+  async exportStages(
+    stages: RespMap[Queries.StageRecordQuery]["stageRecords"]["nodes"],
+  ): Promise<ExportResult> {
+    for (const stage of stages) {
+      await this.mongoDb.collection("stages").updateOne({
+        "stage.id": stage.id,
+      }, {
+        $set: stage,
+      }, {
+        upsert: true,
+      });
+    }
+
+    return {
+      status: "success",
+    };
+  }
+}

src/exporters/stat.ink.ts

@@ -1,5 +1,6 @@
 import {
   AGENT_NAME,
+  COMBINED_VERSION,
   S3SI_VERSION,
   SPLATNET3_STATINK_MAP,
   USERAGENT,
@@ -458,7 +459,7 @@ export class StatInkExporter implements GameExporter {
       ),
 
       agent: AGENT_NAME,
-      agent_version: S3SI_VERSION,
+      agent_version: COMBINED_VERSION,
       agent_variables: {
         "Upload Mode": this.uploadMode,
       },

src/splatnet3.ts

@@ -257,6 +257,12 @@ export class Splatnet3 {
       CoopHistoryQuery,
     };
   }
+
+  async getStageRecords() {
+    const resp = await this.request(Queries.StageRecordQuery);
+
+    return resp;
+  }
 }
 
 function getIdsFromGroups<T extends { id: string }>(

src/state.ts

@@ -30,6 +30,8 @@ export type State = {
   statInkApiKey?: string;
   fileExportPath: string;
   monitorInterval: number;
+  mongoDbUri?: string;
+  splashcatApiKey?: string;
 };
 
 export const DEFAULT_STATE: State = {

src/types.ts

@@ -1,3 +1,4 @@
+import { splatNet3Types } from "../deps.ts";
 import { RankState } from "./state.ts";
 import { Queries } from "./constant.ts";
 export { Queries };
@@ -20,6 +21,7 @@ export type VarsMap = {
   [Queries.myOutfitCommonDataEquipmentsQuery]: [];
   [Queries.HistoryRecordQuery]: [];
   [Queries.ConfigureAnalyticsQuery]: [];
+  [Queries.StageRecordQuery]: [];
 };
 
 export type Image = {
@@ -370,6 +372,9 @@ export type GameExporter = {
   ) => Promise<string[]>;
   exportGame: (game: Game) => Promise<ExportResult>;
   exportSummary?: (summary: Summary) => Promise<ExportResult>;
+  exportStages?: (
+    stages: RespMap[Queries.StageRecordQuery]["stageRecords"]["nodes"],
+  ) => Promise<ExportResult>;
 };
 
 export type BankaraBattleHistories = {
@@ -549,6 +554,7 @@ export type RespMap = {
       xMatchMaxLf: SimpleXRank;
     } | null;
   };
+  [Queries.StageRecordQuery]: splatNet3Types.StageRecordResult;
 };
 export type WeaponWithRatio = {
   weapon: {