catgirl
/
s3si.ts
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 4 Wiki Activity

add stage exporting

main
Rosalina 2023-03-07 10:09:32 -05:00
parent 93b360d5b2
commit 3f31bc7ea9
No known key found for this signature in database
8 changed files with 169 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
deps.ts
View File

@ -13,4 +13,5 @@ export * as path from "https://deno.land/std@0.160.0/path/mod.ts";
export { MultiProgressBar } from "https://deno.land/x/progress@v1.2.8/mod.ts";
export { Mutex } from "https://deno.land/x/semaphore@v1.1.1/mod.ts";
export type { DeepReadonly } from "https://deno.land/x/ts_essentials@v9.1.2/mod.ts";
export * as MongoDB from "npm:mongodb";
export * as MongoDB from "npm:mongodb";
export * as splatNet3Types from "npm:splatnet3-types/splatnet3";

2
s3si.ts
View File

@ -14,6 +14,7 @@ const parseArgs = (args: string[]) => {
"monitor": ["m"],
"skipMode": ["s", "skip-mode"],
"withSummary": "with-summary",
"withStages": "with-stages",
},
});
return parsed;
@ -34,6 +35,7 @@ Options:
--skip-mode <mode>, -s Skip mode (default: null)
("vs", "coop")
--with-summary Include summary in the output
--with-stages Include stage records in the output
--help Show this help message and exit`,
);
Deno.exit(0);

32
scripts/convert-old-mongodb-battles.ts
View File

@ -3,7 +3,7 @@ import { DEFAULT_ENV } from "../src/env.ts";
import { MongoDBExporter } from "../src/exporters/mongodb.ts";
import { FileStateBackend, Profile } from "../src/state.ts";
const OLD_BATTLES_END_DATE = new Date('2023-02-28T03:42:47.000+00:00');
const OLD_BATTLES_END_DATE = new Date("2023-02-28T03:42:47.000+00:00");
const env = DEFAULT_ENV;
const stateBackend = new FileStateBackend("./profile.json");
@ -11,18 +11,18 @@ const profile = new Profile({ stateBackend, env });
await profile.readState();
if (!profile.state.mongoDbUri) {
console.error("MongoDB URI not set");
Deno.exit(1);
console.error("MongoDB URI not set");
Deno.exit(1);
}
const mongoDbClient = new MongoDB.MongoClient(profile.state.mongoDbUri);
const battlesCollection = mongoDbClient.db("splashcat").collection("battles");
const filter = {
"exportDate": {
"$lte": OLD_BATTLES_END_DATE,
},
"gameId": undefined,
"exportDate": {
"$lte": OLD_BATTLES_END_DATE,
},
"gameId": undefined,
};
const cursor = battlesCollection.find(filter);
@ -32,16 +32,18 @@ const oldDocuments = await battlesCollection.countDocuments(filter);
console.log(`Found ${oldDocuments} old battles to update...`);
for await (const doc of cursor) {
const { splatNetData, _id } = doc;
const { splatNetData, _id } = doc;
const splatNetId = splatNetData.id;
const uniqueId = MongoDBExporter.getGameId(splatNetId);
const splatNetId = splatNetData.id;
const uniqueId = MongoDBExporter.getGameId(splatNetId);
await battlesCollection.updateOne({ _id }, { "$set": {
gameId: uniqueId,
}});
await battlesCollection.updateOne({ _id }, {
"$set": {
gameId: uniqueId,
},
});
console.log(`Updated ${splatNetId} to ${uniqueId}`);
console.log(`Updated ${splatNetId} to ${uniqueId}`);
}
console.log("Done!")
console.log("Done!");

28
scripts/fix-stored-dates.ts
View File

@ -9,17 +9,17 @@ const profile = new Profile({ stateBackend, env });
await profile.readState();
if (!profile.state.mongoDbUri) {
console.error("MongoDB URI not set");
Deno.exit(1);
console.error("MongoDB URI not set");
Deno.exit(1);
}
const mongoDbClient = new MongoDB.MongoClient(profile.state.mongoDbUri);
const battlesCollection = mongoDbClient.db("splashcat").collection("battles");
const filter = {
"splatNetData.playedTime": {
$type: "string"
}
"splatNetData.playedTime": {
$type: "string",
},
};
const cursor = battlesCollection.find(filter);
@ -29,13 +29,19 @@ const oldDocuments = await battlesCollection.countDocuments(filter);
console.log(`Found ${oldDocuments} old battles to update...`);
for await (const doc of cursor) {
const { splatNetData, _id } = doc;
const { splatNetData, _id } = doc;
await battlesCollection.updateOne({ _id }, { "$set": {
"splatNetData.playedTime": new Date(splatNetData.playedTime),
}});
await battlesCollection.updateOne({ _id }, {
"$set": {
"splatNetData.playedTime": new Date(splatNetData.playedTime),
},
});
console.log(`Updated ${splatNetData.playedTime} to ${new Date(splatNetData.playedTime)}`);
console.log(
`Updated ${splatNetData.playedTime} to ${new Date(
splatNetData.playedTime,
)}`,
);
}
console.log("Done!")
console.log("Done!");

30
src/app.ts
View File

@ -17,6 +17,7 @@ export type Opts = {
noProgress: boolean;
monitor: boolean;
withSummary: boolean;
withStages: boolean;
skipMode?: string;
cache?: Cache;
stateBackend?: StateBackend;
@ -29,6 +30,7 @@ export const DEFAULT_OPTS: Opts = {
noProgress: false,
monitor: false,
withSummary: false,
withStages: true,
env: DEFAULT_ENV,
};
@ -310,6 +312,34 @@ export class App {
throw errors[0];
}
}
const stageExporters = exporters.filter((e) => e.exportStages);
if (!this.opts.withStages || stageExporters.length === 0) {
this.env.logger.log("Skip exporting stages.");
} else {
const stageRecords = await splatnet.getStageRecords();
await Promise.all(
stageExporters.map((e) =>
showError(
this.env,
e.exportStages!(stageRecords.stageRecords.nodes),
).then((result) => {
if (result.status === "success") {
this.env.logger.log(`Exported stages to ${result.url}`);
} else if (result.status === "skip") {
this.env.logger.log(`Skipped exporting stages to ${e.name}`);
} else {
const _never: never = result;
}
})
.catch((err) => {
errors.push(err);
this.env.logger.error(`\nFailed to export to ${e.name}:`, err);
})
),
);
}
}
async monitor() {
while (true) {

164
src/exporters/mongodb.ts
View File

@ -1,101 +1,113 @@
import { MongoDB } from "../../deps.ts";
import { AGENT_VERSION, NSOAPP_VERSION, S3SI_VERSION } from "../constant.ts";
import { CoopHistoryDetail, ExportResult, Game, GameExporter, Summary, VsHistoryDetail } from "../types.ts";
import {
CoopHistoryDetail,
ExportResult,
Game,
GameExporter,
Summary,
VsHistoryDetail,
} from "../types.ts";
import { parseHistoryDetailId } from "../utils.ts";
export class MongoDBExporter implements GameExporter {
name = "mongodb";
mongoDbClient: MongoDB.MongoClient;
mongoDb: MongoDB.Db;
battlesCollection: MongoDB.Collection;
jobsCollection: MongoDB.Collection;
summariesCollection: MongoDB.Collection;
constructor(private mongoDbUri: string) {
this.mongoDbClient = new MongoDB.MongoClient(mongoDbUri);
this.mongoDb = this.mongoDbClient.db("splashcat");
this.battlesCollection = this.mongoDb.collection("battles");
this.jobsCollection = this.mongoDb.collection("jobs");
this.summariesCollection = this.mongoDb.collection("summaries");
}
name = "mongodb";
mongoDbClient: MongoDB.MongoClient;
mongoDb: MongoDB.Db;
battlesCollection: MongoDB.Collection;
jobsCollection: MongoDB.Collection;
summariesCollection: MongoDB.Collection;
constructor(private mongoDbUri: string) {
this.mongoDbClient = new MongoDB.MongoClient(mongoDbUri);
this.mongoDb = this.mongoDbClient.db("splashcat");
this.battlesCollection = this.mongoDb.collection("battles");
this.jobsCollection = this.mongoDb.collection("jobs");
this.summariesCollection = this.mongoDb.collection("summaries");
}
static getGameId(id: string) { // very similar to the file exporter
const { uid, timestamp } = parseHistoryDetailId(id);
static getGameId(id: string) { // very similar to the file exporter
const { uid, timestamp } = parseHistoryDetailId(id);
return `${uid}_${timestamp}Z`;
}
return `${uid}_${timestamp}Z`;
}
async notExported({ type, list }: { type: Game["type"], list: string[] }): Promise<string[]> {
const out: string[] = [];
async notExported(
{ type, list }: { type: Game["type"]; list: string[] },
): Promise<string[]> {
const out: string[] = [];
const collection = type === "CoopInfo" ? this.jobsCollection : this.battlesCollection;
const collection = type === "CoopInfo"
? this.jobsCollection
: this.battlesCollection;
for (const id of list) {
const uniqueId = MongoDBExporter.getGameId(id);
const countNewStorage = await collection.countDocuments({
gameId: uniqueId,
});
for (const id of list) {
const uniqueId = MongoDBExporter.getGameId(id);
const countNewStorage = await collection.countDocuments({
gameId: uniqueId,
});
if (countNewStorage === 0) {
out.push(id);
}
}
if (countNewStorage === 0) {
out.push(id);
}
}
return out;
}
return out;
}
async exportGame(game: Game): Promise<ExportResult> {
const uniqueId = MongoDBExporter.getGameId(game.detail.id);
async exportGame(game: Game): Promise<ExportResult> {
const uniqueId = MongoDBExporter.getGameId(game.detail.id);
const common = {
// this seems like useful data to store...
// loosely modeled after FileExporterTypeCommon
nsoVersion: NSOAPP_VERSION,
agentVersion: AGENT_VERSION,
s3siVersion: S3SI_VERSION,
exportDate: new Date(),
};
const common = {
// this seems like useful data to store...
// loosely modeled after FileExporterTypeCommon
nsoVersion: NSOAPP_VERSION,
agentVersion: AGENT_VERSION,
s3siVersion: S3SI_VERSION,
exportDate: new Date(),
};
const splatNetData = {
...game.detail,
playedTime: new Date(game.detail.playedTime),
};
const splatNetData = {
...game.detail,
playedTime: new Date(game.detail.playedTime),
};
const body:
{
data: Game,
splatNetData: Omit<(VsHistoryDetail | CoopHistoryDetail), "playedTime"> & { playedTime: Date },
gameId: string,
} & typeof common = {
...common,
data: game,
splatNetData,
gameId: uniqueId,
};
const body: {
data: Game;
splatNetData:
& Omit<(VsHistoryDetail | CoopHistoryDetail), "playedTime">
& { playedTime: Date };
gameId: string;
} & typeof common = {
...common,
data: game,
splatNetData,
gameId: uniqueId,
};
const isJob = game.type === "CoopInfo";
const isJob = game.type === "CoopInfo";
const collection = isJob ? this.jobsCollection : this.battlesCollection;
const collection = isJob ? this.jobsCollection : this.battlesCollection;
const result = await collection.insertOne(body);
const result = await collection.insertOne(body);
const objectId = result.insertedId;
const objectId = result.insertedId;
return {
status: "success",
url: `https://new.splatoon.catgirlin.space/battle/${objectId.toString()}`,
};
}
return {
status: "success",
url: `https://new.splatoon.catgirlin.space/battle/${objectId.toString()}`,
};
}
async exportSummary(summary: Summary): Promise<ExportResult> {
const id = summary.uid;
async exportSummary(summary: Summary): Promise<ExportResult> {
const id = summary.uid;
await this.summariesCollection.insertOne({
summaryId: id,
...summary,
});
await this.summariesCollection.insertOne({
summaryId: id,
...summary,
});
return {
status: "success",
};
}
return {
status: "success",
};
}
}

6
src/splatnet3.ts
View File

@ -257,6 +257,12 @@ export class Splatnet3 {
CoopHistoryQuery,
};
}
async getStageRecords() {
const resp = await this.request(Queries.StageRecordQuery);
return resp;
}
}
function getIdsFromGroups<T extends { id: string }>(

7
src/types.ts
View File

@ -1,3 +1,4 @@
import { splatNet3Types } from "../deps.ts";
import { RankState } from "./state.ts";
export enum Queries {
@ -15,6 +16,7 @@ export enum Queries {
myOutfitCommonDataEquipmentsQuery = "d29cd0c2b5e6bac90dd5b817914832f8",
HistoryRecordQuery = "32b6771f94083d8f04848109b7300af5",
ConfigureAnalyticsQuery = "f8ae00773cc412a50dd41a6d9a159ddd",
StageRecordQuery = "f08a932d533845dde86e674e03bbb7d3",
}
export type VarsMap = {
[Queries.HomeQuery]: [];
@ -34,6 +36,7 @@ export type VarsMap = {
[Queries.myOutfitCommonDataEquipmentsQuery]: [];
[Queries.HistoryRecordQuery]: [];
[Queries.ConfigureAnalyticsQuery]: [];
[Queries.StageRecordQuery]: [];
};
export type Image = {
@ -371,6 +374,9 @@ export type GameExporter = {
) => Promise<string[]>;
exportGame: (game: Game) => Promise<ExportResult>;
exportSummary?: (summary: Summary) => Promise<ExportResult>;
exportStages?: (
stages: RespMap[Queries.StageRecordQuery]["stageRecords"]["nodes"],
) => Promise<ExportResult>;
};
export type BankaraBattleHistories = {
@ -550,6 +556,7 @@ export type RespMap = {
xMatchMaxLf: SimpleXRank;
} | null;
};
[Queries.StageRecordQuery]: splatNet3Types.StageRecordResult;
};
export type WeaponWithRatio = {
weapon: {