commit splashcat stuff

deno.lock

@@ -71,7 +71,7 @@
   },
   "npm": {
     "specifiers": {
-      "mongodb": "mongodb@5.1.0",
+      "mongodb": "mongodb@5.5.0",
       "splatnet3-types": "splatnet3-types@0.2.20230227204004"
     },
     "packages": {
@@ -94,6 +94,10 @@
         "integrity": "sha512-y09gBGusgHtinMon/GVbv1J6FrXhnr/+6hqLlSmEFzkz6PodqF6TxjyvfvY3AfO+oG1mgUtbC86xSbOlwvM62Q==",
         "dependencies": {}
       },
+      "bson@5.3.0": {
+        "integrity": "sha512-ukmCZMneMlaC5ebPHXIkP8YJzNl5DC41N5MAIvKDqLggdao342t4McltoJBQfQya/nHBWAcSsYRqlXPoQkTJag==",
+        "dependencies": {}
+      },
       "ip@2.0.0": {
         "integrity": "sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==",
         "dependencies": {}
@@ -118,6 +122,15 @@
           "socks": "socks@2.7.1"
         }
       },
+      "mongodb@5.5.0": {
+        "integrity": "sha512-XgrkUgAAdfnZKQfk5AsYL8j7O99WHd4YXPxYxnh8dZxD+ekYWFRA3JktUsBnfg+455Smf75/+asoU/YLwNGoQQ==",
+        "dependencies": {
+          "bson": "bson@5.3.0",
+          "mongodb-connection-string-url": "mongodb-connection-string-url@2.6.0",
+          "saslprep": "saslprep@1.0.3",
+          "socks": "socks@2.7.1"
+        }
+      },
       "punycode@2.3.0": {
         "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==",
         "dependencies": {}

scripts/deno.lock

@@ -84,5 +84,104 @@
     "https://deno.land/x/ts_essentials@v9.1.2/lib/mod.ts": "d7e44a25aa621425ffd118a0210a492c5c354411018e2db648a68614d5901f5b",
     "https://deno.land/x/ts_essentials@v9.1.2/lib/types.ts": "7ee99797a880948c07020e90d569ca3c5d465c378949262110283aa7856f5603",
     "https://deno.land/x/ts_essentials@v9.1.2/mod.ts": "ffae461c16d4a1bf24c2179582ab8d5c81ad0df61e4ae2fba51ef5e5bdf90345"
+  },
+  "npm": {
+    "specifiers": {
+      "mongodb": "mongodb@5.1.0",
+      "splatnet3-types": "splatnet3-types@0.2.20230227204004"
+    },
+    "packages": {
+      "@types/node@18.14.2": {
+        "integrity": "sha512-1uEQxww3DaghA0RxqHx0O0ppVlo43pJhepY51OxuQIKHpjbnYLA7vcdwioNPzIqmC2u3I/dmylcqjlh0e7AyUA==",
+        "dependencies": {}
+      },
+      "@types/webidl-conversions@7.0.0": {
+        "integrity": "sha512-xTE1E+YF4aWPJJeUzaZI5DRntlkY3+BCVJi0axFptnjGmAoWxkyREIh/XMrfxVLejwQxMCfDXdICo0VLxThrog==",
+        "dependencies": {}
+      },
+      "@types/whatwg-url@8.2.2": {
+        "integrity": "sha512-FtQu10RWgn3D9U4aazdwIE2yzphmTJREDqNdODHrbrZmmMqI0vMheC/6NE/J1Yveaj8H+ela+YwWTjq5PGmuhA==",
+        "dependencies": {
+          "@types/node": "@types/node@18.14.2",
+          "@types/webidl-conversions": "@types/webidl-conversions@7.0.0"
+        }
+      },
+      "bson@5.0.1": {
+        "integrity": "sha512-y09gBGusgHtinMon/GVbv1J6FrXhnr/+6hqLlSmEFzkz6PodqF6TxjyvfvY3AfO+oG1mgUtbC86xSbOlwvM62Q==",
+        "dependencies": {}
+      },
+      "ip@2.0.0": {
+        "integrity": "sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==",
+        "dependencies": {}
+      },
+      "memory-pager@1.5.0": {
+        "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==",
+        "dependencies": {}
+      },
+      "mongodb-connection-string-url@2.6.0": {
+        "integrity": "sha512-WvTZlI9ab0QYtTYnuMLgobULWhokRjtC7db9LtcVfJ+Hsnyr5eo6ZtNAt3Ly24XZScGMelOcGtm7lSn0332tPQ==",
+        "dependencies": {
+          "@types/whatwg-url": "@types/whatwg-url@8.2.2",
+          "whatwg-url": "whatwg-url@11.0.0"
+        }
+      },
+      "mongodb@5.1.0": {
+        "integrity": "sha512-qgKb7y+EI90y4weY3z5+lIgm8wmexbonz0GalHkSElQXVKtRuwqXuhXKccyvIjXCJVy9qPV82zsinY0W1FBnJw==",
+        "dependencies": {
+          "bson": "bson@5.0.1",
+          "mongodb-connection-string-url": "mongodb-connection-string-url@2.6.0",
+          "saslprep": "saslprep@1.0.3",
+          "socks": "socks@2.7.1"
+        }
+      },
+      "punycode@2.3.0": {
+        "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==",
+        "dependencies": {}
+      },
+      "saslprep@1.0.3": {
+        "integrity": "sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag==",
+        "dependencies": {
+          "sparse-bitfield": "sparse-bitfield@3.0.3"
+        }
+      },
+      "smart-buffer@4.2.0": {
+        "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==",
+        "dependencies": {}
+      },
+      "socks@2.7.1": {
+        "integrity": "sha512-7maUZy1N7uo6+WVEX6psASxtNlKaNVMlGQKkG/63nEDdLOWNbiUMoLK7X4uYoLhQstau72mLgfEWcXcwsaHbYQ==",
+        "dependencies": {
+          "ip": "ip@2.0.0",
+          "smart-buffer": "smart-buffer@4.2.0"
+        }
+      },
+      "sparse-bitfield@3.0.3": {
+        "integrity": "sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==",
+        "dependencies": {
+          "memory-pager": "memory-pager@1.5.0"
+        }
+      },
+      "splatnet3-types@0.2.20230227204004": {
+        "integrity": "sha512-FAY6pbUcrp5O8c49BNXSKxoyM3UlCrRx2AtA9Y3qlvqOLdHqwxtzcdzbk1b1hRam8ZcrxRzE/ii6ESRiPIAnZw==",
+        "dependencies": {}
+      },
+      "tr46@3.0.0": {
+        "integrity": "sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==",
+        "dependencies": {
+          "punycode": "punycode@2.3.0"
+        }
+      },
+      "webidl-conversions@7.0.0": {
+        "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==",
+        "dependencies": {}
+      },
+      "whatwg-url@11.0.0": {
+        "integrity": "sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==",
+        "dependencies": {
+          "tr46": "tr46@3.0.0",
+          "webidl-conversions": "webidl-conversions@7.0.0"
+        }
+      }
+    }
   }
-}
+}

scripts/send-mongodb-battles-to-splashcat.ts

@@ -0,0 +1,77 @@
+import { MongoDB } from "../deps.ts";
+import { DEFAULT_ENV } from "../src/env.ts";
+import { MongoDBExporter } from "../src/exporters/mongodb.ts";
+import { FileStateBackend, Profile } from "../src/state.ts";
+
+const env = DEFAULT_ENV;
+const stateBackend = new FileStateBackend("./profile.json");
+const profile = new Profile({ stateBackend, env });
+await profile.readState();
+
+if (!profile.state.mongoDbUri) {
+  console.error("MongoDB URI not set");
+  Deno.exit(1);
+}
+
+const mongoDbClient = new MongoDB.MongoClient(profile.state.mongoDbUri);
+const battlesCollection = mongoDbClient.db("splashcat").collection("battles");
+
+const cursor = battlesCollection.find();
+
+const oldDocuments = await battlesCollection.countDocuments();
+
+console.log(`Found ${oldDocuments} old battles to upload...`);
+
+let count = 0;
+
+const erroredBattles = [];
+
+for await (const doc of cursor) {
+  const { splatNetData, _id } = doc;
+
+  // start time for performance tracking, needs to be very accurate
+  const startTime = new Date();
+
+  splatNetData.playedTime = splatNetData.playedTime.toISOString();
+
+  const response = await fetch("http://127.0.0.1:8000/battles/api/upload/", {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+      "Authorization": `Bearer ${profile.state.splashcatApiKey}`,
+    },
+    body: JSON.stringify({
+      "data_type": "splatnet3",
+      "battle": splatNetData,
+    })
+  })
+
+  if (!response.ok) {
+    console.error(`Failed to upload ${splatNetData.id}`);
+    erroredBattles.push({
+      id: doc.gameId,
+      error: await response.text(),
+    });
+  }
+
+  // end time for performance tracking, needs to be very accurate
+  const endTime = new Date();
+  const timeTaken = endTime.getTime() - startTime.getTime();
+  
+  console.log(`Uploaded ${splatNetData.id} (${timeTaken}ms)`);
+  count++;
+  console.log(`Uploaded ${count}/${oldDocuments} battles`)
+
+  if (count % 100 === 0) {
+    console.log("Updating error logs...");
+    if (erroredBattles.length > 0) {
+      await Deno.writeFile("./errored-battles.json", new TextEncoder().encode(JSON.stringify(erroredBattles, null, "\t")));
+    }
+  }
+}
+
+console.log("Done!");
+
+if (erroredBattles.length > 0) {
+  await Deno.writeFile("./errored-battles.json", new TextEncoder().encode(JSON.stringify(erroredBattles, null, 2)));
+}

src/state.ts

@@ -31,6 +31,7 @@ export type State = {
   fileExportPath: string;
   monitorInterval: number;
   mongoDbUri?: string;
+  splashcatApiKey?: string;
 };
 
 export const DEFAULT_STATE: State = {