From f04f88a02a51efc8405f5ee1b18c591d2f078c0d Mon Sep 17 00:00:00 2001 From: kitt Date: Wed, 24 May 2023 23:41:09 -0400 Subject: [PATCH] commit splashcat stuff --- deno.lock | 15 ++- scripts/deno.lock | 101 ++++++++++++++++++- scripts/send-mongodb-battles-to-splashcat.ts | 77 ++++++++++++++ src/state.ts | 1 + 4 files changed, 192 insertions(+), 2 deletions(-) create mode 100644 scripts/send-mongodb-battles-to-splashcat.ts diff --git a/deno.lock b/deno.lock index 1bbf05d..0edec13 100644 --- a/deno.lock +++ b/deno.lock @@ -71,7 +71,7 @@ }, "npm": { "specifiers": { - "mongodb": "mongodb@5.1.0", + "mongodb": "mongodb@5.5.0", "splatnet3-types": "splatnet3-types@0.2.20230227204004" }, "packages": { @@ -94,6 +94,10 @@ "integrity": "sha512-y09gBGusgHtinMon/GVbv1J6FrXhnr/+6hqLlSmEFzkz6PodqF6TxjyvfvY3AfO+oG1mgUtbC86xSbOlwvM62Q==", "dependencies": {} }, + "bson@5.3.0": { + "integrity": "sha512-ukmCZMneMlaC5ebPHXIkP8YJzNl5DC41N5MAIvKDqLggdao342t4McltoJBQfQya/nHBWAcSsYRqlXPoQkTJag==", + "dependencies": {} + }, "ip@2.0.0": { "integrity": "sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==", "dependencies": {} @@ -118,6 +122,15 @@ "socks": "socks@2.7.1" } }, + "mongodb@5.5.0": { + "integrity": "sha512-XgrkUgAAdfnZKQfk5AsYL8j7O99WHd4YXPxYxnh8dZxD+ekYWFRA3JktUsBnfg+455Smf75/+asoU/YLwNGoQQ==", + "dependencies": { + "bson": "bson@5.3.0", + "mongodb-connection-string-url": "mongodb-connection-string-url@2.6.0", + "saslprep": "saslprep@1.0.3", + "socks": "socks@2.7.1" + } + }, "punycode@2.3.0": { "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==", "dependencies": {} diff --git a/scripts/deno.lock b/scripts/deno.lock index df6de4a..ce8d4c5 100644 --- a/scripts/deno.lock +++ b/scripts/deno.lock @@ -84,5 +84,104 @@ "https://deno.land/x/ts_essentials@v9.1.2/lib/mod.ts": "d7e44a25aa621425ffd118a0210a492c5c354411018e2db648a68614d5901f5b", "https://deno.land/x/ts_essentials@v9.1.2/lib/types.ts": "7ee99797a880948c07020e90d569ca3c5d465c378949262110283aa7856f5603", "https://deno.land/x/ts_essentials@v9.1.2/mod.ts": "ffae461c16d4a1bf24c2179582ab8d5c81ad0df61e4ae2fba51ef5e5bdf90345" + }, + "npm": { + "specifiers": { + "mongodb": "mongodb@5.1.0", + "splatnet3-types": "splatnet3-types@0.2.20230227204004" + }, + "packages": { + "@types/node@18.14.2": { + "integrity": "sha512-1uEQxww3DaghA0RxqHx0O0ppVlo43pJhepY51OxuQIKHpjbnYLA7vcdwioNPzIqmC2u3I/dmylcqjlh0e7AyUA==", + "dependencies": {} + }, + "@types/webidl-conversions@7.0.0": { + "integrity": "sha512-xTE1E+YF4aWPJJeUzaZI5DRntlkY3+BCVJi0axFptnjGmAoWxkyREIh/XMrfxVLejwQxMCfDXdICo0VLxThrog==", + "dependencies": {} + }, + "@types/whatwg-url@8.2.2": { + "integrity": "sha512-FtQu10RWgn3D9U4aazdwIE2yzphmTJREDqNdODHrbrZmmMqI0vMheC/6NE/J1Yveaj8H+ela+YwWTjq5PGmuhA==", + "dependencies": { + "@types/node": "@types/node@18.14.2", + "@types/webidl-conversions": "@types/webidl-conversions@7.0.0" + } + }, + "bson@5.0.1": { + "integrity": "sha512-y09gBGusgHtinMon/GVbv1J6FrXhnr/+6hqLlSmEFzkz6PodqF6TxjyvfvY3AfO+oG1mgUtbC86xSbOlwvM62Q==", + "dependencies": {} + }, + "ip@2.0.0": { + "integrity": "sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==", + "dependencies": {} + }, + "memory-pager@1.5.0": { + "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==", + "dependencies": {} + }, + "mongodb-connection-string-url@2.6.0": { + "integrity": "sha512-WvTZlI9ab0QYtTYnuMLgobULWhokRjtC7db9LtcVfJ+Hsnyr5eo6ZtNAt3Ly24XZScGMelOcGtm7lSn0332tPQ==", + "dependencies": { + "@types/whatwg-url": "@types/whatwg-url@8.2.2", + "whatwg-url": "whatwg-url@11.0.0" + } + }, + "mongodb@5.1.0": { + "integrity": "sha512-qgKb7y+EI90y4weY3z5+lIgm8wmexbonz0GalHkSElQXVKtRuwqXuhXKccyvIjXCJVy9qPV82zsinY0W1FBnJw==", + "dependencies": { + "bson": "bson@5.0.1", + "mongodb-connection-string-url": "mongodb-connection-string-url@2.6.0", + "saslprep": "saslprep@1.0.3", + "socks": "socks@2.7.1" + } + }, + "punycode@2.3.0": { + "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==", + "dependencies": {} + }, + "saslprep@1.0.3": { + "integrity": "sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag==", + "dependencies": { + "sparse-bitfield": "sparse-bitfield@3.0.3" + } + }, + "smart-buffer@4.2.0": { + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "dependencies": {} + }, + "socks@2.7.1": { + "integrity": "sha512-7maUZy1N7uo6+WVEX6psASxtNlKaNVMlGQKkG/63nEDdLOWNbiUMoLK7X4uYoLhQstau72mLgfEWcXcwsaHbYQ==", + "dependencies": { + "ip": "ip@2.0.0", + "smart-buffer": "smart-buffer@4.2.0" + } + }, + "sparse-bitfield@3.0.3": { + "integrity": "sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==", + "dependencies": { + "memory-pager": "memory-pager@1.5.0" + } + }, + "splatnet3-types@0.2.20230227204004": { + "integrity": "sha512-FAY6pbUcrp5O8c49BNXSKxoyM3UlCrRx2AtA9Y3qlvqOLdHqwxtzcdzbk1b1hRam8ZcrxRzE/ii6ESRiPIAnZw==", + "dependencies": {} + }, + "tr46@3.0.0": { + "integrity": "sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==", + "dependencies": { + "punycode": "punycode@2.3.0" + } + }, + "webidl-conversions@7.0.0": { + "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", + "dependencies": {} + }, + "whatwg-url@11.0.0": { + "integrity": "sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==", + "dependencies": { + "tr46": "tr46@3.0.0", + "webidl-conversions": "webidl-conversions@7.0.0" + } + } + } } -} \ No newline at end of file +} diff --git a/scripts/send-mongodb-battles-to-splashcat.ts b/scripts/send-mongodb-battles-to-splashcat.ts new file mode 100644 index 0000000..af90dc8 --- /dev/null +++ b/scripts/send-mongodb-battles-to-splashcat.ts @@ -0,0 +1,77 @@ +import { MongoDB } from "../deps.ts"; +import { DEFAULT_ENV } from "../src/env.ts"; +import { MongoDBExporter } from "../src/exporters/mongodb.ts"; +import { FileStateBackend, Profile } from "../src/state.ts"; + +const env = DEFAULT_ENV; +const stateBackend = new FileStateBackend("./profile.json"); +const profile = new Profile({ stateBackend, env }); +await profile.readState(); + +if (!profile.state.mongoDbUri) { + console.error("MongoDB URI not set"); + Deno.exit(1); +} + +const mongoDbClient = new MongoDB.MongoClient(profile.state.mongoDbUri); +const battlesCollection = mongoDbClient.db("splashcat").collection("battles"); + +const cursor = battlesCollection.find(); + +const oldDocuments = await battlesCollection.countDocuments(); + +console.log(`Found ${oldDocuments} old battles to upload...`); + +let count = 0; + +const erroredBattles = []; + +for await (const doc of cursor) { + const { splatNetData, _id } = doc; + + // start time for performance tracking, needs to be very accurate + const startTime = new Date(); + + splatNetData.playedTime = splatNetData.playedTime.toISOString(); + + const response = await fetch("http://127.0.0.1:8000/battles/api/upload/", { + method: "POST", + headers: { + "Content-Type": "application/json", + "Authorization": `Bearer ${profile.state.splashcatApiKey}`, + }, + body: JSON.stringify({ + "data_type": "splatnet3", + "battle": splatNetData, + }) + }) + + if (!response.ok) { + console.error(`Failed to upload ${splatNetData.id}`); + erroredBattles.push({ + id: doc.gameId, + error: await response.text(), + }); + } + + // end time for performance tracking, needs to be very accurate + const endTime = new Date(); + const timeTaken = endTime.getTime() - startTime.getTime(); + + console.log(`Uploaded ${splatNetData.id} (${timeTaken}ms)`); + count++; + console.log(`Uploaded ${count}/${oldDocuments} battles`) + + if (count % 100 === 0) { + console.log("Updating error logs..."); + if (erroredBattles.length > 0) { + await Deno.writeFile("./errored-battles.json", new TextEncoder().encode(JSON.stringify(erroredBattles, null, "\t"))); + } + } +} + +console.log("Done!"); + +if (erroredBattles.length > 0) { + await Deno.writeFile("./errored-battles.json", new TextEncoder().encode(JSON.stringify(erroredBattles, null, 2))); +} diff --git a/src/state.ts b/src/state.ts index a58e144..c03edac 100644 --- a/src/state.ts +++ b/src/state.ts @@ -31,6 +31,7 @@ export type State = { fileExportPath: string; monitorInterval: number; mongoDbUri?: string; + splashcatApiKey?: string; }; export const DEFAULT_STATE: State = {