catgirl
/
s3si.ts
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 4 Wiki Activity

commit splashcat stuff

main
Rosalina 2023-05-24 23:41:09 -04:00
parent 32d4586cce
commit f04f88a02a
No known key found for this signature in database
4 changed files with 192 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
deno.lock
View File

@ -71,7 +71,7 @@
},
"npm": {
"specifiers": {
"mongodb": "mongodb@5.1.0",
"mongodb": "mongodb@5.5.0",
"splatnet3-types": "splatnet3-types@0.2.20230227204004"
},
"packages": {
@ -94,6 +94,10 @@
"integrity": "sha512-y09gBGusgHtinMon/GVbv1J6FrXhnr/+6hqLlSmEFzkz6PodqF6TxjyvfvY3AfO+oG1mgUtbC86xSbOlwvM62Q==",
"dependencies": {}
},
"bson@5.3.0": {
"integrity": "sha512-ukmCZMneMlaC5ebPHXIkP8YJzNl5DC41N5MAIvKDqLggdao342t4McltoJBQfQya/nHBWAcSsYRqlXPoQkTJag==",
"dependencies": {}
},
"ip@2.0.0": {
"integrity": "sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==",
"dependencies": {}
@ -118,6 +122,15 @@
"socks": "socks@2.7.1"
}
},
"mongodb@5.5.0": {
"integrity": "sha512-XgrkUgAAdfnZKQfk5AsYL8j7O99WHd4YXPxYxnh8dZxD+ekYWFRA3JktUsBnfg+455Smf75/+asoU/YLwNGoQQ==",
"dependencies": {
"bson": "bson@5.3.0",
"mongodb-connection-string-url": "mongodb-connection-string-url@2.6.0",
"saslprep": "saslprep@1.0.3",
"socks": "socks@2.7.1"
}
},
"punycode@2.3.0": {
"integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==",
"dependencies": {}

99
scripts/deno.lock
View File

@ -84,5 +84,104 @@
"https://deno.land/x/ts_essentials@v9.1.2/lib/mod.ts": "d7e44a25aa621425ffd118a0210a492c5c354411018e2db648a68614d5901f5b",
"https://deno.land/x/ts_essentials@v9.1.2/lib/types.ts": "7ee99797a880948c07020e90d569ca3c5d465c378949262110283aa7856f5603",
"https://deno.land/x/ts_essentials@v9.1.2/mod.ts": "ffae461c16d4a1bf24c2179582ab8d5c81ad0df61e4ae2fba51ef5e5bdf90345"
},
"npm": {
"specifiers": {
"mongodb": "mongodb@5.1.0",
"splatnet3-types": "splatnet3-types@0.2.20230227204004"
},
"packages": {
"@types/node@18.14.2": {
"integrity": "sha512-1uEQxww3DaghA0RxqHx0O0ppVlo43pJhepY51OxuQIKHpjbnYLA7vcdwioNPzIqmC2u3I/dmylcqjlh0e7AyUA==",
"dependencies": {}
},
"@types/webidl-conversions@7.0.0": {
"integrity": "sha512-xTE1E+YF4aWPJJeUzaZI5DRntlkY3+BCVJi0axFptnjGmAoWxkyREIh/XMrfxVLejwQxMCfDXdICo0VLxThrog==",
"dependencies": {}
},
"@types/whatwg-url@8.2.2": {
"integrity": "sha512-FtQu10RWgn3D9U4aazdwIE2yzphmTJREDqNdODHrbrZmmMqI0vMheC/6NE/J1Yveaj8H+ela+YwWTjq5PGmuhA==",
"dependencies": {
"@types/node": "@types/node@18.14.2",
"@types/webidl-conversions": "@types/webidl-conversions@7.0.0"
}
},
"bson@5.0.1": {
"integrity": "sha512-y09gBGusgHtinMon/GVbv1J6FrXhnr/+6hqLlSmEFzkz6PodqF6TxjyvfvY3AfO+oG1mgUtbC86xSbOlwvM62Q==",
"dependencies": {}
},
"ip@2.0.0": {
"integrity": "sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==",
"dependencies": {}
},
"memory-pager@1.5.0": {
"integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==",
"dependencies": {}
},
"mongodb-connection-string-url@2.6.0": {
"integrity": "sha512-WvTZlI9ab0QYtTYnuMLgobULWhokRjtC7db9LtcVfJ+Hsnyr5eo6ZtNAt3Ly24XZScGMelOcGtm7lSn0332tPQ==",
"dependencies": {
"@types/whatwg-url": "@types/whatwg-url@8.2.2",
"whatwg-url": "whatwg-url@11.0.0"
}
},
"mongodb@5.1.0": {
"integrity": "sha512-qgKb7y+EI90y4weY3z5+lIgm8wmexbonz0GalHkSElQXVKtRuwqXuhXKccyvIjXCJVy9qPV82zsinY0W1FBnJw==",
"dependencies": {
"bson": "bson@5.0.1",
"mongodb-connection-string-url": "mongodb-connection-string-url@2.6.0",
"saslprep": "saslprep@1.0.3",
"socks": "socks@2.7.1"
}
},
"punycode@2.3.0": {
"integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==",
"dependencies": {}
},
"saslprep@1.0.3": {
"integrity": "sha512-/MY/PEMbk2SuY5sScONwhUDsV2p77Znkb/q3nSVstq/yQzYJOH/Azh29p9oJLsl3LnQwSvZDKagDGBsBwSooag==",
"dependencies": {
"sparse-bitfield": "sparse-bitfield@3.0.3"
}
},
"smart-buffer@4.2.0": {
"integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==",
"dependencies": {}
},
"socks@2.7.1": {
"integrity": "sha512-7maUZy1N7uo6+WVEX6psASxtNlKaNVMlGQKkG/63nEDdLOWNbiUMoLK7X4uYoLhQstau72mLgfEWcXcwsaHbYQ==",
"dependencies": {
"ip": "ip@2.0.0",
"smart-buffer": "smart-buffer@4.2.0"
}
},
"sparse-bitfield@3.0.3": {
"integrity": "sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==",
"dependencies": {
"memory-pager": "memory-pager@1.5.0"
}
},
"splatnet3-types@0.2.20230227204004": {
"integrity": "sha512-FAY6pbUcrp5O8c49BNXSKxoyM3UlCrRx2AtA9Y3qlvqOLdHqwxtzcdzbk1b1hRam8ZcrxRzE/ii6ESRiPIAnZw==",
"dependencies": {}
},
"tr46@3.0.0": {
"integrity": "sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==",
"dependencies": {
"punycode": "punycode@2.3.0"
}
},
"webidl-conversions@7.0.0": {
"integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==",
"dependencies": {}
},
"whatwg-url@11.0.0": {
"integrity": "sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==",
"dependencies": {
"tr46": "tr46@3.0.0",
"webidl-conversions": "webidl-conversions@7.0.0"
}
}
}
}
}

77
scripts/send-mongodb-battles-to-splashcat.ts Normal file
View File

@ -0,0 +1,77 @@
import { MongoDB } from "../deps.ts";
import { DEFAULT_ENV } from "../src/env.ts";
import { MongoDBExporter } from "../src/exporters/mongodb.ts";
import { FileStateBackend, Profile } from "../src/state.ts";
const env = DEFAULT_ENV;
const stateBackend = new FileStateBackend("./profile.json");
const profile = new Profile({ stateBackend, env });
await profile.readState();
if (!profile.state.mongoDbUri) {
console.error("MongoDB URI not set");
Deno.exit(1);
}
const mongoDbClient = new MongoDB.MongoClient(profile.state.mongoDbUri);
const battlesCollection = mongoDbClient.db("splashcat").collection("battles");
const cursor = battlesCollection.find();
const oldDocuments = await battlesCollection.countDocuments();
console.log(`Found ${oldDocuments} old battles to upload...`);
let count = 0;
const erroredBattles = [];
for await (const doc of cursor) {
const { splatNetData, _id } = doc;
// start time for performance tracking, needs to be very accurate
const startTime = new Date();
splatNetData.playedTime = splatNetData.playedTime.toISOString();
const response = await fetch("http://127.0.0.1:8000/battles/api/upload/", {
method: "POST",
headers: {
"Content-Type": "application/json",
"Authorization": `Bearer ${profile.state.splashcatApiKey}`,
},
body: JSON.stringify({
"data_type": "splatnet3",
"battle": splatNetData,
})
})
if (!response.ok) {
console.error(`Failed to upload ${splatNetData.id}`);
erroredBattles.push({
id: doc.gameId,
error: await response.text(),
});
}
// end time for performance tracking, needs to be very accurate
const endTime = new Date();
const timeTaken = endTime.getTime() - startTime.getTime();
console.log(`Uploaded ${splatNetData.id} (${timeTaken}ms)`);
count++;
console.log(`Uploaded ${count}/${oldDocuments} battles`)
if (count % 100 === 0) {
console.log("Updating error logs...");
if (erroredBattles.length > 0) {
await Deno.writeFile("./errored-battles.json", new TextEncoder().encode(JSON.stringify(erroredBattles, null, "\t")));
}
}
}
console.log("Done!");
if (erroredBattles.length > 0) {
await Deno.writeFile("./errored-battles.json", new TextEncoder().encode(JSON.stringify(erroredBattles, null, 2)));
}

1
src/state.ts
View File

@ -31,6 +31,7 @@ export type State = {
fileExportPath: string;
monitorInterval: number;
mongoDbUri?: string;
splashcatApiKey?: string;
};
export const DEFAULT_STATE: State = {