add early MongoDB exporter

mongodb exporter can now identify what already exists and is compatible with the previous format of splashcat's db.

deno.lock

@@ -72,5 +72,91 @@
     "https://deno.land/x/ts_essentials@v9.1.2/lib/mod.ts": "d7e44a25aa621425ffd118a0210a492c5c354411018e2db648a68614d5901f5b",
     "https://deno.land/x/ts_essentials@v9.1.2/lib/types.ts": "7ee99797a880948c07020e90d569ca3c5d465c378949262110283aa7856f5603",
     "https://deno.land/x/ts_essentials@v9.1.2/mod.ts": "ffae461c16d4a1bf24c2179582ab8d5c81ad0df61e4ae2fba51ef5e5bdf90345"
+  },
+  "npm": {
+    "specifiers": { "mongodb": "mongodb@5.1.0" },
+    "packages": {
+      "@types/node@18.14.2": {
+        "integrity": "sha512-1uEQxww3DaghA0RxqHx0O0ppVlo43pJhepY51OxuQIKHpjbnYLA7vcdwioNPzIqmC2u3I/dmylcqjlh0e7AyUA==",
+        "dependencies": {}
+      },
+      "@types/webidl-conversions@7.0.0": {
+        "integrity": "sha512-xTE1E+YF4aWPJJeUzaZI5DRntlkY3+BCVJi0axFptnjGmAoWxkyREIh/XMrfxVLejwQxMCfDXdICo0VLxThrog==",
+        "dependencies": {}
+      },
+      "@types/whatwg-url@8.2.2": {
+        "integrity": "sha512-FtQu10RWgn3D9U4aazdwIE2yzphmTJREDqNdODHrbrZmmMqI0vMheC/6NE/J1Yveaj8H+ela+YwWTjq5PGmuhA==",
+        "dependencies": {
+          "@types/node": "@types/node@18.14.2",
+          "@types/webidl-conversions": "@types/webidl-conversions@7.0.0"
+        }
+      },
+      "bson@5.0.1": {
+        "integrity": "sha512-y09gBGusgHtinMon/GVbv1J6FrXhnr/+6hqLlSmEFzkz6PodqF6TxjyvfvY3AfO+oG1mgUtbC86xSbOlwvM62Q==",
+        "dependencies": {}
+      },
+      "ip@2.0.0": {
+        "integrity": "sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==",
+        "dependencies": {}
+      },
+      "memory-pager@1.5.0": {
+        "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==",
+        "dependencies": {}
+      },
+      "mongodb-connection-string-url@2.6.0": {
+        "integrity": "sha512-WvTZlI9ab0QYtTYnuMLgobULWhokRjtC7db9LtcVfJ+Hsnyr5eo6ZtNAt3Ly24XZScGMelOcGtm7lSn0332tPQ==",
+        "dependencies": {
+          "@types/whatwg-url": "@types/whatwg-url@8.2.2",
+          "whatwg-url": "whatwg-url@11.0.0"
+        }
+      },
+      "mongodb@5.1.0": {
+        "integrity": "sha512-qgKb7y+EI90y4weY3z5+lIgm8wmexbonz0GalHkSElQXVKtRuwqXuhXKccyvIjXCJVy9qPV82zsinY0W1FBnJw==",
+        "dependencies": {
+          "bson": "bson@5.0.1",
+          "mongodb-connection-string-url": "mongodb-connection-string-url@2.6.0",
+          "saslprep": "saslprep@1.0.3",
+          "socks": "socks@2.7.1"
+        }
+      },
+      "punycode@2.3.0": {
+        "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==",
+        "dependencies": {}
+      },
+      "saslprep@1.0.3": {
+        "integrity": "sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag==",
+        "dependencies": { "sparse-bitfield": "sparse-bitfield@3.0.3" }
+      },
+      "smart-buffer@4.2.0": {
+        "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==",
+        "dependencies": {}
+      },
+      "socks@2.7.1": {
+        "integrity": "sha512-7maUZy1N7uo6+WVEX6psASxtNlKaNVMlGQKkG/63nEDdLOWNbiUMoLK7X4uYoLhQstau72mLgfEWcXcwsaHbYQ==",
+        "dependencies": {
+          "ip": "ip@2.0.0",
+          "smart-buffer": "smart-buffer@4.2.0"
+        }
+      },
+      "sparse-bitfield@3.0.3": {
+        "integrity": "sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==",
+        "dependencies": { "memory-pager": "memory-pager@1.5.0" }
+      },
+      "tr46@3.0.0": {
+        "integrity": "sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==",
+        "dependencies": { "punycode": "punycode@2.3.0" }
+      },
+      "webidl-conversions@7.0.0": {
+        "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==",
+        "dependencies": {}
+      },
+      "whatwg-url@11.0.0": {
+        "integrity": "sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==",
+        "dependencies": {
+          "tr46": "tr46@3.0.0",
+          "webidl-conversions": "webidl-conversions@7.0.0"
+        }
+      }
+    }
   }
 }

deps.ts

@@ -13,3 +13,4 @@ export * as path from "https://deno.land/std@0.160.0/path/mod.ts";
 export { MultiProgressBar } from "https://deno.land/x/progress@v1.2.8/mod.ts";
 export { Mutex } from "https://deno.land/x/semaphore@v1.1.1/mod.ts";
 export type { DeepReadonly } from "https://deno.land/x/ts_essentials@v9.1.2/mod.ts";
+export * as MongoDB from "npm:mongodb";

src/app.ts

@@ -9,6 +9,7 @@ import { FileExporter } from "./exporters/file.ts";
 import { delay, showError } from "./utils.ts";
 import { GameFetcher } from "./GameFetcher.ts";
 import { DEFAULT_ENV, Env } from "./env.ts";
+import { MongoDBExporter } from "./exporters/mongodb.ts";
 
 export type Opts = {
   profilePath: string;
@@ -115,6 +116,25 @@ export class App {
       out.push(new FileExporter(state.fileExportPath));
     }
 
+    if (exporters.includes("mongodb")) {
+      if (!state.mongoDbUri) {
+        const uri = (await this.env.prompts.prompt(
+          "MongoDB URI is not set. Please enter below.",
+        )).trim();
+        if (!uri) {
+          this.env.logger.error("MongoDB URI is required.");
+          Deno.exit(1);
+        }
+        await this.profile.writeState({
+          ...state,
+          mongoDbUri: uri,
+        });
+      }
+      out.push(
+        new MongoDBExporter(this.profile.state.mongoDbUri!),
+      );
+    }
+
     return out;
   }
   exporterProgress(title: string) {

src/exporters/mongodb.ts

@@ -1,3 +1,50 @@
-import { GameExporter } from "../types.ts";
+import { MongoDB } from "../../deps.ts";
+import { Game, GameExporter } from "../types.ts";
+import { parseHistoryDetailId } from "../utils.ts";
 
-export class MongoDBExporter implements GameExporter {}
+export class MongoDBExporter implements GameExporter {
+	name = "mongodb";
+	mongoDbClient: MongoDB.MongoClient;
+	mongoDb: MongoDB.Db;
+	battlesCollection: MongoDB.Collection;
+	jobsCollection: MongoDB.Collection;
+	constructor(private mongoDbUri: string) {
+		this.mongoDbClient = new MongoDB.MongoClient(mongoDbUri);
+		this.mongoDb = this.mongoDbClient.db("splashcat");
+		this.battlesCollection = this.mongoDb.collection("battles");
+		this.jobsCollection = this.mongoDb.collection("jobs");
+	}
+
+	getGameId(id: string) { // very similar to the file exporter
+		const { uid, timestamp } = parseHistoryDetailId(id);
+
+		return `${uid}_${timestamp}Z`;
+	}
+
+	async notExported({ type, list }: { type: Game["type"], list: string[] }): Promise<string[]> {
+		const out: string[] = [];
+
+		const collection = type === "CoopInfo" ? this.jobsCollection : this.battlesCollection;
+
+		for (const id of list) {
+			// countOldStorage can be removed later eventually when all old documents
+			// are gone from SplatNet 3
+			const countOldStorage = await collection.countDocuments({
+				splatNetData: {
+					id: id,
+				}
+			});
+
+			const uniqueId = this.getGameId(id);
+			const countNewStorage = await collection.countDocuments({
+				gameId: uniqueId,
+			});
+
+			if (countOldStorage === 0 && countNewStorage === 0) {
+				out.push(id);
+			}
+		}
+
+		return out;
+	}
+}

src/state.ts

@@ -30,6 +30,7 @@ export type State = {
   statInkApiKey?: string;
   fileExportPath: string;
   monitorInterval: number;
+  mongoDbUri?: string;
 };
 
 export const DEFAULT_STATE: State = {