diff --git a/s3si.ts b/s3si.ts index 12d2cf7..b7e6bdc 100644 --- a/s3si.ts +++ b/s3si.ts @@ -4,7 +4,7 @@ import { flags } from "./deps.ts"; const parseArgs = (args: string[]) => { const parsed = flags.parse(args, { - string: ["profilePath", "exporter"], + string: ["profilePath", "exporter", "skipMode"], boolean: ["help", "noProgress", "monitor"], alias: { "help": "h", @@ -12,6 +12,7 @@ const parseArgs = (args: string[]) => { "exporter": ["e"], "noProgress": ["n", "no-progress"], "monitor": ["m"], + "skipMode": ["s", "skip-mode"], }, }); return parsed; @@ -28,6 +29,9 @@ Options: Multiple exporters can be separated by commas (e.g. "stat.ink,file") --no-progress, -n Disable progress bar + --monitor, -m Monitor mode + --skip-mode , -s Skip mode (default: null) + ("vs", "coop") --help Show this help message and exit`, ); Deno.exit(0); diff --git a/src/app.ts b/src/app.ts index 5733617..85a660c 100644 --- a/src/app.ts +++ b/src/app.ts @@ -10,21 +10,27 @@ import { getBankaraBattleHistories, getBattleDetail, getBattleList, + getCoopDetail, + getCoopHistories, isTokenExpired, } from "./splatnet3.ts"; import { - BattleExporter, + BattleListNode, + BattleListType, ChallengeProgress, + CoopInfo, + CoopListNode, + Game, + GameExporter, HistoryGroups, - VsBattle, - VsHistoryDetail, + VsInfo, } from "./types.ts"; import { Cache, FileCache, MemoryCache } from "./cache.ts"; import { StatInkExporter } from "./exporters/stat.ink.ts"; import { FileExporter } from "./exporters/file.ts"; import { - battleId, delay, + gameId, readline, RecoverableError, retryRecoverableError, @@ -36,6 +42,7 @@ export type Opts = { exporter: string; noProgress: boolean; monitor: boolean; + skipMode?: string; cache?: Cache; stateBackend?: StateBackend; }; @@ -48,14 +55,16 @@ export const DEFAULT_OPTS: Opts = { }; /** - * Fetch battle and cache it. + * Fetch game and cache it. */ -class BattleFetcher { +class GameFetcher { state: State; cache: Cache; lock: Record = {}; bankaraLock = new Mutex(); - bankaraHistory?: HistoryGroups["nodes"]; + bankaraHistory?: HistoryGroups["nodes"]; + coopLock = new Mutex(); + coopHistory?: HistoryGroups["nodes"]; constructor( { cache = new MemoryCache(), state }: { state: State; cache?: Cache }, @@ -64,7 +73,7 @@ class BattleFetcher { this.cache = cache; } private async getLock(id: string): Promise { - const bid = await battleId(id); + const bid = await gameId(id); let cur = this.lock[bid]; if (!cur) { @@ -74,6 +83,7 @@ class BattleFetcher { return cur; } + getBankaraHistory() { return this.bankaraLock.use(async () => { if (this.bankaraHistory) { @@ -90,8 +100,44 @@ class BattleFetcher { return this.bankaraHistory; }); } - async getBattleMetaById(id: string): Promise> { - const bid = await battleId(id); + getCoopHistory() { + return this.coopLock.use(async () => { + if (this.coopHistory) { + return this.coopHistory; + } + + const { coopResult: { historyGroups } } = await getCoopHistories( + this.state, + ); + + this.coopHistory = historyGroups.nodes; + + return this.coopHistory; + }); + } + async getCoopMetaById(id: string): Promise> { + const coopHistory = await this.getCoopHistory(); + const group = coopHistory.find((i) => + i.historyDetails.nodes.some((i) => i.id === id) + ); + + if (!group) { + return { + type: "CoopInfo", + listNode: null, + }; + } + + const listNode = group.historyDetails.nodes.find((i) => i.id === id) ?? + null; + + return { + type: "CoopInfo", + listNode, + }; + } + async getBattleMetaById(id: string): Promise> { + const bid = await gameId(id); const bankaraHistory = await this.getBankaraHistory(); const group = bankaraHistory.find((i) => i.historyDetails.nodes.some((i) => i._bid === bid) @@ -99,7 +145,7 @@ class BattleFetcher { if (!group) { return { - type: "VsBattle", + type: "VsInfo", challengeProgress: null, bankaraMatchChallenge: null, listNode: null, @@ -127,39 +173,68 @@ class BattleFetcher { } return { - type: "VsBattle", + type: "VsInfo", bankaraMatchChallenge, listNode, challengeProgress, }; } - async getBattleDetail(id: string): Promise { + async cacheDetail( + id: string, + getter: () => Promise, + ): Promise { const lock = await this.getLock(id); return lock.use(async () => { - const cached = await this.cache.read(id); + const cached = await this.cache.read(id); if (cached) { return cached; } - const detail = (await getBattleDetail(this.state, id)) - .vsHistoryDetail; + const detail = await getter(); await this.cache.write(id, detail); return detail; }); } - async fetchBattle(id: string): Promise { - const detail = await this.getBattleDetail(id); + fetch(type: Game["type"], id: string): Promise { + switch (type) { + case "VsInfo": + return this.fetchBattle(id); + case "CoopInfo": + return this.fetchCoop(id); + default: + throw new Error(`Unknown game type: ${type}`); + } + } + async fetchBattle(id: string): Promise { + const detail = await this.cacheDetail( + id, + () => getBattleDetail(this.state, id).then((r) => r.vsHistoryDetail), + ); const metadata = await this.getBattleMetaById(id); - const battle: VsBattle = { + const game: VsInfo = { ...metadata, detail, }; - return battle; + return game; + } + async fetchCoop(id: string): Promise { + const detail = await this.cacheDetail( + id, + () => getCoopDetail(this.state, id).then((r) => r.coopHistoryDetail), + ); + const metadata = await this.getCoopMetaById(id); + + const game: CoopInfo = { + ...metadata, + detail, + }; + + return game; } } @@ -203,9 +278,18 @@ export class App { await this.writeState(DEFAULT_STATE); } } - async getExporters(): Promise[]> { + getSkipMode(): ("vs" | "coop")[] { + const mode = this.opts.skipMode; + if (mode === "vs") { + return ["vs"]; + } else if (mode === "coop") { + return ["coop"]; + } + return []; + } + async getExporters(): Promise { const exporters = this.opts.exporter.split(","); - const out: BattleExporter[] = []; + const out: GameExporter[] = []; if (exporters.includes("stat.ink")) { if (!this.state.statInkApiKey) { @@ -237,46 +321,58 @@ export class App { exportOnce() { return retryRecoverableError(() => this._exportOnce(), this.recoveryToken); } - async _exportOnce() { + exporterProgress(title: string) { const bar = !this.opts.noProgress ? new MultiProgressBar({ - title: "Export battles", + title, display: "[:bar] :text :percent :time eta: :eta :completed/:total", }) : undefined; - try { - const exporters = await this.getExporters(); + const allProgress: Record = {}; + const redraw = (name: string, progress: Progress) => { + allProgress[name] = progress; + bar?.render( + Object.entries(allProgress).map(([name, progress]) => ({ + completed: progress.current, + total: progress.total, + text: name, + })), + ); + }; + const endBar = () => { + bar?.end(); + }; - const fetcher = new BattleFetcher({ + return { redraw, endBar }; + } + async _exportOnce() { + const exporters = await this.getExporters(); + const skipMode = this.getSkipMode(); + const stats: Record = Object.fromEntries( + exporters.map((e) => [e.name, 0]), + ); + + if (skipMode.includes("vs")) { + console.log("Skip exporting VS games."); + } else { + console.log("Fetching battle list..."); + const gameList = await getBattleList(this.state); + + const { redraw, endBar } = this.exporterProgress("Export games"); + const fetcher = new GameFetcher({ cache: this.opts.cache ?? new FileCache(this.state.cacheDir), state: this.state, }); - console.log("Fetching battle list..."); - const battleList = await getBattleList(this.state); - - const allProgress: Record = {}; - const redraw = (name: string, progress: Progress) => { - allProgress[name] = progress; - bar?.render( - Object.entries(allProgress).map(([name, progress]) => ({ - completed: progress.current, - total: progress.total, - text: name, - })), - ); - }; - const stats: Record = Object.fromEntries( - exporters.map((e) => [e.name, 0]), - ); await Promise.all( exporters.map((e) => showError( - this.exportBattleList({ + this.exportGameList({ + type: "VsInfo", fetcher, exporter: e, - battleList, + gameList, onStep: (progress) => redraw(e.name, progress), }) .then((count) => { @@ -289,18 +385,55 @@ export class App { ), ); - bar?.end(); - - console.log( - `Exported ${ - Object.entries(stats) - .map(([name, count]) => `${name}: ${count}`) - .join(", ") - }`, - ); - } finally { - bar?.end(); + endBar(); } + + if (skipMode.includes("coop")) { + console.log("Skip exporting Coop games."); + } else { + console.log("Fetching coop battle list..."); + const coopBattleList = await getBattleList( + this.state, + BattleListType.Coop, + ); + + const { redraw, endBar } = this.exporterProgress("Export games"); + const fetcher = new GameFetcher({ + cache: this.opts.cache ?? new FileCache(this.state.cacheDir), + state: this.state, + }); + + await Promise.all( + // TODO: remove this filter when stat.ink support coop export + exporters.filter((e) => e.name !== "stat.ink").map((e) => + showError( + this.exportGameList({ + type: "CoopInfo", + fetcher, + exporter: e, + gameList: coopBattleList, + onStep: (progress) => redraw(e.name, progress), + }) + .then((count) => { + stats[e.name] = count; + }), + ) + .catch((err) => { + console.error(`\nFailed to export to ${e.name}:`, err); + }) + ), + ); + + endBar(); + } + + console.log( + `Exported ${ + Object.entries(stats) + .map(([name, count]) => `${name}: ${count}`) + .join(", ") + }`, + ); } async monitor() { while (true) { @@ -376,26 +509,26 @@ export class App { } } /** - * Export battle list. + * Export game list. * * @param fetcher BattleFetcher * @param exporter BattleExporter - * @param battleList ID list of battles, sorted by date, newest first - * @param onStep Callback function called when a battle is exported + * @param gameList ID list of games, sorted by date, newest first + * @param onStep Callback function called when a game is exported */ - async exportBattleList( - { - fetcher, - exporter, - battleList, - onStep, - }: { - fetcher: BattleFetcher; - exporter: BattleExporter; - battleList: string[]; - onStep?: (progress: Progress) => void; - }, - ): Promise { + async exportGameList({ + type, + fetcher, + exporter, + gameList, + onStep, + }: { + type: Game["type"]; + exporter: GameExporter; + fetcher: GameFetcher; + gameList: string[]; + onStep: (progress: Progress) => void; + }) { let exported = 0; onStep?.({ @@ -403,11 +536,17 @@ export class App { total: 1, }); - const workQueue = [...await exporter.notExported(battleList)].reverse(); + const workQueue = [ + ...await exporter.notExported({ + type, + list: gameList, + }), + ] + .reverse(); - const step = async (battle: string) => { - const detail = await fetcher.fetchBattle(battle); - await exporter.exportBattle(detail); + const step = async (id: string) => { + const detail = await fetcher.fetch(type, id); + await exporter.exportGame(detail); exported += 1; onStep?.({ current: exported, diff --git a/src/constant.ts b/src/constant.ts index be2c209..8cad493 100644 --- a/src/constant.ts +++ b/src/constant.ts @@ -1,7 +1,7 @@ import type { StatInkPostBody, VsHistoryDetail } from "./types.ts"; export const AGENT_NAME = "s3si.ts"; -export const S3SI_VERSION = "0.1.7"; +export const S3SI_VERSION = "0.1.8"; export const NSOAPP_VERSION = "2.3.1"; export const WEB_VIEW_VERSION = "1.0.0-216d0219"; export const S3SI_LINK = "https://github.com/spacemeowx2/s3si.ts" diff --git a/src/exporters/file.ts b/src/exporters/file.ts index 2af9b87..234781a 100644 --- a/src/exporters/file.ts +++ b/src/exporters/file.ts @@ -1,14 +1,14 @@ -import { BattleExporter, VsBattle } from "../types.ts"; +import { CoopInfo, GameExporter, VsInfo } from "../types.ts"; import { path } from "../../deps.ts"; import { NSOAPP_VERSION, S3SI_VERSION } from "../constant.ts"; -import { parseVsHistoryDetailId } from "../utils.ts"; +import { parseHistoryDetailId } from "../utils.ts"; export type FileExporterType = { type: "VS" | "COOP"; nsoVersion: string; s3siVersion: string; exportTime: string; - data: VsBattle; + data: VsInfo | CoopInfo; }; /** @@ -26,27 +26,27 @@ function replacer(key: string, value: unknown): unknown { * This is useful for debugging. It will write each battle detail to a file. * Timestamp is used as filename. Example: 20210101T000000Z.json */ -export class FileExporter implements BattleExporter { +export class FileExporter implements GameExporter { name = "file"; constructor(private exportPath: string) { } getFilenameById(id: string) { - const { uid, timestamp } = parseVsHistoryDetailId(id); + const { uid, timestamp } = parseHistoryDetailId(id); return `${uid}_${timestamp}Z.json`; } - async exportBattle(battle: VsBattle) { + async exportGame(info: VsInfo | CoopInfo) { await Deno.mkdir(this.exportPath, { recursive: true }); - const filename = this.getFilenameById(battle.detail.id); + const filename = this.getFilenameById(info.detail.id); const filepath = path.join(this.exportPath, filename); const body: FileExporterType = { - type: "VS", + type: info.type === "VsInfo" ? "VS" : "COOP", nsoVersion: NSOAPP_VERSION, s3siVersion: S3SI_VERSION, exportTime: new Date().toISOString(), - data: battle, + data: info, }; await Deno.writeTextFile( @@ -54,7 +54,7 @@ export class FileExporter implements BattleExporter { JSON.stringify(body, replacer), ); } - async notExported(list: string[]): Promise { + async notExported({ list }: { list: string[] }): Promise { const out: string[] = []; for (const id of list) { diff --git a/src/exporters/stat.ink.ts b/src/exporters/stat.ink.ts index 64db4d7..c49cade 100644 --- a/src/exporters/stat.ink.ts +++ b/src/exporters/stat.ink.ts @@ -5,17 +5,18 @@ import { USERAGENT, } from "../constant.ts"; import { - BattleExporter, + CoopInfo, + GameExporter, StatInkPlayer, StatInkPostBody, StatInkStage, - VsBattle, VsHistoryDetail, + VsInfo, VsPlayer, } from "../types.ts"; import { base64, msgpack } from "../../deps.ts"; import { APIError } from "../APIError.ts"; -import { battleId, cache } from "../utils.ts"; +import { cache, gameId } from "../utils.ts"; const S3S_NAMESPACE = "b3a2dbf5-2c09-4792-b78c-00b548b70aeb"; @@ -40,7 +41,7 @@ const getStage = cache(_getStage); * * This is the default exporter. It will upload each battle detail to stat.ink. */ -export class StatInkExporter implements BattleExporter { +export class StatInkExporter implements GameExporter { name = "stat.ink"; constructor(private statInkApiKey: string, private uploadMode: string) { @@ -54,8 +55,12 @@ export class StatInkExporter implements BattleExporter { "Authorization": `Bearer ${this.statInkApiKey}`, }; } - async exportBattle(battle: VsBattle) { - const body = await this.mapBattle(battle); + async exportGame(game: VsInfo | CoopInfo) { + if (game.type === "CoopInfo") { + // TODO: support coop + return; + } + const body = await this.mapBattle(game); const resp = await fetch("https://stat.ink/api/v3/battle", { method: "POST", @@ -86,7 +91,7 @@ export class StatInkExporter implements BattleExporter { }); } } - async notExported(list: string[]): Promise { + async notExported({ list }: { list: string[] }): Promise { const uuid = await (await fetch("https://stat.ink/api/v3/s3s/uuid-list", { headers: this.requestHeaders(), })).json(); @@ -94,8 +99,8 @@ export class StatInkExporter implements BattleExporter { const out: string[] = []; for (const id of list) { - const s3sId = await battleId(id, S3S_NAMESPACE); - const s3siId = await battleId(id); + const s3sId = await gameId(id, S3S_NAMESPACE); + const s3siId = await gameId(id); if (!uuid.includes(s3sId) && !uuid.includes(s3siId)) { out.push(id); @@ -166,7 +171,7 @@ export class StatInkExporter implements BattleExporter { } async mapBattle( { challengeProgress, bankaraMatchChallenge, listNode, detail: vsDetail }: - VsBattle, + VsInfo, ): Promise { const { knockout, @@ -185,7 +190,7 @@ export class StatInkExporter implements BattleExporter { const startedAt = Math.floor(new Date(playedTime).getTime() / 1000); const result: StatInkPostBody = { - uuid: await battleId(vsDetail.id), + uuid: await gameId(vsDetail.id), lobby: this.mapLobby(vsDetail), rule: SPLATNET3_STATINK_MAP.RULE[vsDetail.vsRule.rule], stage: await this.mapStage(vsDetail), diff --git a/src/splatnet3.ts b/src/splatnet3.ts index 509ce41..aee4e1e 100644 --- a/src/splatnet3.ts +++ b/src/splatnet3.ts @@ -13,7 +13,7 @@ import { RespMap, VarsMap, } from "./types.ts"; -import { battleId } from "./utils.ts"; +import { gameId } from "./utils.ts"; async function request( state: State, @@ -90,7 +90,9 @@ export async function checkToken(state: State) { } } -function getIdsFromGroups({ historyGroups }: { historyGroups: HistoryGroups }) { +function getIdsFromGroups( + { historyGroups }: { historyGroups: HistoryGroups }, +) { return historyGroups.nodes.flatMap((i) => i.historyDetails.nodes).map((i) => i.id ); @@ -112,6 +114,9 @@ const BATTLE_LIST_TYPE_MAP: Record< [BattleListType.Private]: (state: State) => request(state, Queries.PrivateBattleHistoriesQuery) .then((r) => getIdsFromGroups(r.privateBattleHistories)), + [BattleListType.Coop]: (state: State) => + request(state, Queries.CoopHistoryQuery) + .then((r) => getIdsFromGroups(r.coopResult)), }; export async function getBattleList( @@ -134,12 +139,31 @@ export function getBattleDetail( ); } +export function getCoopDetail( + state: State, + id: string, +) { + return request( + state, + Queries.CoopHistoryDetailQuery, + { + coopHistoryDetailId: id, + }, + ); +} + export async function getBankaraBattleHistories(state: State) { const resp = await request(state, Queries.BankaraBattleHistoriesQuery); for (const i of resp.bankaraBattleHistories.historyGroups.nodes) { for (const j of i.historyDetails.nodes) { - j._bid = await battleId(j.id); + j._bid = await gameId(j.id); } } return resp; } + +export async function getCoopHistories(state: State) { + const resp = await request(state, Queries.CoopHistoryQuery); + + return resp; +} diff --git a/src/types.ts b/src/types.ts index 5deee1e..57d5073 100644 --- a/src/types.ts +++ b/src/types.ts @@ -46,11 +46,15 @@ export type BattleListNode = { udemae: string; judgement: "LOSE" | "WIN" | "DEEMED_LOSE" | "EXEMPTED_LOSE"; }; -export type HistoryGroups = { +export type CoopListNode = { + id: string; +}; +export type HistoryGroups = { nodes: { bankaraMatchChallenge: null | BankaraMatchChallenge; + historyDetails: { - nodes: BattleListNode[]; + nodes: T[]; }; }[]; }; @@ -96,13 +100,20 @@ export type ChallengeProgress = { loseCount: number; }; // With challenge info -export type VsBattle = { - type: "VsBattle"; +export type VsInfo = { + type: "VsInfo"; listNode: null | BattleListNode; bankaraMatchChallenge: null | BankaraMatchChallenge; challengeProgress: null | ChallengeProgress; detail: VsHistoryDetail; }; +// Salmon run +export type CoopInfo = { + type: "CoopInfo"; + listNode: null | CoopListNode; + detail: CoopHistoryDetail; +}; +export type Game = VsInfo | CoopInfo; export type VsHistoryDetail = { id: string; vsRule: { @@ -138,16 +149,21 @@ export type VsHistoryDetail = { awards: { name: string; rank: string }[]; duration: number; }; +export type CoopHistoryDetail = { + id: string; +}; -export type BattleExporter< - D extends { - // type is seful when you implement more than one BattleExporter on the same class +export type GameExporter< + T extends { + // type is seful when you implement more than one GameExporter on the same class type: string; - }, + } = Game, > = { name: string; - notExported: (list: string[]) => Promise; - exportBattle: (detail: D) => Promise; + notExported: ( + { type, list }: { type: T["type"]; list: string[] }, + ) => Promise; + exportGame: (game: T) => Promise; }; export type RespMap = { @@ -171,29 +187,35 @@ export type RespMap = { }; [Queries.LatestBattleHistoriesQuery]: { latestBattleHistories: { - historyGroups: HistoryGroups; + historyGroups: HistoryGroups; }; }; [Queries.RegularBattleHistoriesQuery]: { regularBattleHistories: { - historyGroups: HistoryGroups; + historyGroups: HistoryGroups; }; }; [Queries.BankaraBattleHistoriesQuery]: { bankaraBattleHistories: { - historyGroups: HistoryGroups; + historyGroups: HistoryGroups; }; }; [Queries.PrivateBattleHistoriesQuery]: { privateBattleHistories: { - historyGroups: HistoryGroups; + historyGroups: HistoryGroups; }; }; [Queries.VsHistoryDetailQuery]: { vsHistoryDetail: VsHistoryDetail; }; - [Queries.CoopHistoryQuery]: Record; - [Queries.CoopHistoryDetailQuery]: Record; + [Queries.CoopHistoryQuery]: { + coopResult: { + historyGroups: HistoryGroups; + }; + }; + [Queries.CoopHistoryDetailQuery]: { + coopHistoryDetail: CoopHistoryDetail; + }; }; export type GraphQLResponse = { data: T; @@ -208,6 +230,7 @@ export enum BattleListType { Regular, Bankara, Private, + Coop, } export type StatInkPlayer = { diff --git a/src/utils.ts b/src/utils.ts index 2fe4c1a..987d8ab 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -83,11 +83,11 @@ export async function showError(p: Promise): Promise { } /** - * @param id id of VeVsHistoryDetail + * @param id id of VsHistoryDetail or CoopHistoryDetail * @param namespace uuid namespace * @returns */ -export function battleId( +export function gameId( id: string, namespace = S3SI_NAMESPACE, ): Promise { @@ -96,22 +96,38 @@ export function battleId( return uuid.v5.generate(namespace, tsUuid); } -export function parseVsHistoryDetailId(id: string) { +/** + * @param id VsHistoryDetail id or CoopHistoryDetail id + */ +//CoopHistoryDetail-u-quoeuj7rhknjq3jkanmm:20221022T065633_25287bf9-d9a8-42b0-b070-e938da103547 +export function parseHistoryDetailId(id: string) { const plainText = new TextDecoder().decode(base64.decode(id)); - const re = /VsHistoryDetail-([a-z0-9-]+):(\w+):(\d{8}T\d{6})_([0-9a-f-]{36})/; - if (!re.test(plainText)) { - throw new Error(`Invalid battle ID: ${plainText}`); + const vsRE = + /VsHistoryDetail-([a-z0-9-]+):(\w+):(\d{8}T\d{6})_([0-9a-f-]{36})/; + const coopRE = /CoopHistoryDetail-([a-z0-9-]+):(\d{8}T\d{6})_([0-9a-f-]{36})/; + if (vsRE.test(plainText)) { + const [, uid, listType, timestamp, uuid] = plainText.match(vsRE)!; + + return { + type: "VsHistoryDetail", + uid, + listType, + timestamp, + uuid, + }; + } else if (coopRE.test(plainText)) { + const [, uid, timestamp, uuid] = plainText.match(coopRE)!; + + return { + type: "CoopHistoryDetail", + uid, + timestamp, + uuid, + }; + } else { + throw new Error(`Invalid ID: ${plainText}`); } - - const [, uid, listType, timestamp, uuid] = plainText.match(re)!; - - return { - uid, - listType, - timestamp, - uuid, - }; } export const delay = (ms: number) =>