feat: add coop export (v0.1.8)

main
spacemeowx2 2022-10-25 02:45:36 +08:00
parent cd4aa53c7d
commit f30da22d0c
8 changed files with 345 additions and 134 deletions

View File

@ -4,7 +4,7 @@ import { flags } from "./deps.ts";
const parseArgs = (args: string[]) => {
const parsed = flags.parse(args, {
string: ["profilePath", "exporter"],
string: ["profilePath", "exporter", "skipMode"],
boolean: ["help", "noProgress", "monitor"],
alias: {
"help": "h",
@ -12,6 +12,7 @@ const parseArgs = (args: string[]) => {
"exporter": ["e"],
"noProgress": ["n", "no-progress"],
"monitor": ["m"],
"skipMode": ["s", "skip-mode"],
},
});
return parsed;
@ -28,6 +29,9 @@ Options:
Multiple exporters can be separated by commas
(e.g. "stat.ink,file")
--no-progress, -n Disable progress bar
--monitor, -m Monitor mode
--skip-mode <mode>, -s Skip mode (default: null)
("vs", "coop")
--help Show this help message and exit`,
);
Deno.exit(0);

View File

@ -10,21 +10,27 @@ import {
getBankaraBattleHistories,
getBattleDetail,
getBattleList,
getCoopDetail,
getCoopHistories,
isTokenExpired,
} from "./splatnet3.ts";
import {
BattleExporter,
BattleListNode,
BattleListType,
ChallengeProgress,
CoopInfo,
CoopListNode,
Game,
GameExporter,
HistoryGroups,
VsBattle,
VsHistoryDetail,
VsInfo,
} from "./types.ts";
import { Cache, FileCache, MemoryCache } from "./cache.ts";
import { StatInkExporter } from "./exporters/stat.ink.ts";
import { FileExporter } from "./exporters/file.ts";
import {
battleId,
delay,
gameId,
readline,
RecoverableError,
retryRecoverableError,
@ -36,6 +42,7 @@ export type Opts = {
exporter: string;
noProgress: boolean;
monitor: boolean;
skipMode?: string;
cache?: Cache;
stateBackend?: StateBackend;
};
@ -48,14 +55,16 @@ export const DEFAULT_OPTS: Opts = {
};
/**
* Fetch battle and cache it.
* Fetch game and cache it.
*/
class BattleFetcher {
class GameFetcher {
state: State;
cache: Cache;
lock: Record<string, Mutex | undefined> = {};
bankaraLock = new Mutex();
bankaraHistory?: HistoryGroups["nodes"];
bankaraHistory?: HistoryGroups<BattleListNode>["nodes"];
coopLock = new Mutex();
coopHistory?: HistoryGroups<CoopListNode>["nodes"];
constructor(
{ cache = new MemoryCache(), state }: { state: State; cache?: Cache },
@ -64,7 +73,7 @@ class BattleFetcher {
this.cache = cache;
}
private async getLock(id: string): Promise<Mutex> {
const bid = await battleId(id);
const bid = await gameId(id);
let cur = this.lock[bid];
if (!cur) {
@ -74,6 +83,7 @@ class BattleFetcher {
return cur;
}
getBankaraHistory() {
return this.bankaraLock.use(async () => {
if (this.bankaraHistory) {
@ -90,8 +100,44 @@ class BattleFetcher {
return this.bankaraHistory;
});
}
async getBattleMetaById(id: string): Promise<Omit<VsBattle, "detail">> {
const bid = await battleId(id);
getCoopHistory() {
return this.coopLock.use(async () => {
if (this.coopHistory) {
return this.coopHistory;
}
const { coopResult: { historyGroups } } = await getCoopHistories(
this.state,
);
this.coopHistory = historyGroups.nodes;
return this.coopHistory;
});
}
async getCoopMetaById(id: string): Promise<Omit<CoopInfo, "detail">> {
const coopHistory = await this.getCoopHistory();
const group = coopHistory.find((i) =>
i.historyDetails.nodes.some((i) => i.id === id)
);
if (!group) {
return {
type: "CoopInfo",
listNode: null,
};
}
const listNode = group.historyDetails.nodes.find((i) => i.id === id) ??
null;
return {
type: "CoopInfo",
listNode,
};
}
async getBattleMetaById(id: string): Promise<Omit<VsInfo, "detail">> {
const bid = await gameId(id);
const bankaraHistory = await this.getBankaraHistory();
const group = bankaraHistory.find((i) =>
i.historyDetails.nodes.some((i) => i._bid === bid)
@ -99,7 +145,7 @@ class BattleFetcher {
if (!group) {
return {
type: "VsBattle",
type: "VsInfo",
challengeProgress: null,
bankaraMatchChallenge: null,
listNode: null,
@ -127,39 +173,68 @@ class BattleFetcher {
}
return {
type: "VsBattle",
type: "VsInfo",
bankaraMatchChallenge,
listNode,
challengeProgress,
};
}
async getBattleDetail(id: string): Promise<VsHistoryDetail> {
async cacheDetail<T>(
id: string,
getter: () => Promise<T>,
): Promise<T> {
const lock = await this.getLock(id);
return lock.use(async () => {
const cached = await this.cache.read<VsHistoryDetail>(id);
const cached = await this.cache.read<T>(id);
if (cached) {
return cached;
}
const detail = (await getBattleDetail(this.state, id))
.vsHistoryDetail;
const detail = await getter();
await this.cache.write(id, detail);
return detail;
});
}
async fetchBattle(id: string): Promise<VsBattle> {
const detail = await this.getBattleDetail(id);
fetch(type: Game["type"], id: string): Promise<Game> {
switch (type) {
case "VsInfo":
return this.fetchBattle(id);
case "CoopInfo":
return this.fetchCoop(id);
default:
throw new Error(`Unknown game type: ${type}`);
}
}
async fetchBattle(id: string): Promise<VsInfo> {
const detail = await this.cacheDetail(
id,
() => getBattleDetail(this.state, id).then((r) => r.vsHistoryDetail),
);
const metadata = await this.getBattleMetaById(id);
const battle: VsBattle = {
const game: VsInfo = {
...metadata,
detail,
};
return battle;
return game;
}
async fetchCoop(id: string): Promise<CoopInfo> {
const detail = await this.cacheDetail(
id,
() => getCoopDetail(this.state, id).then((r) => r.coopHistoryDetail),
);
const metadata = await this.getCoopMetaById(id);
const game: CoopInfo = {
...metadata,
detail,
};
return game;
}
}
@ -203,9 +278,18 @@ export class App {
await this.writeState(DEFAULT_STATE);
}
}
async getExporters(): Promise<BattleExporter<VsBattle>[]> {
getSkipMode(): ("vs" | "coop")[] {
const mode = this.opts.skipMode;
if (mode === "vs") {
return ["vs"];
} else if (mode === "coop") {
return ["coop"];
}
return [];
}
async getExporters(): Promise<GameExporter[]> {
const exporters = this.opts.exporter.split(",");
const out: BattleExporter<VsBattle>[] = [];
const out: GameExporter[] = [];
if (exporters.includes("stat.ink")) {
if (!this.state.statInkApiKey) {
@ -237,24 +321,14 @@ export class App {
exportOnce() {
return retryRecoverableError(() => this._exportOnce(), this.recoveryToken);
}
async _exportOnce() {
exporterProgress(title: string) {
const bar = !this.opts.noProgress
? new MultiProgressBar({
title: "Export battles",
title,
display: "[:bar] :text :percent :time eta: :eta :completed/:total",
})
: undefined;
try {
const exporters = await this.getExporters();
const fetcher = new BattleFetcher({
cache: this.opts.cache ?? new FileCache(this.state.cacheDir),
state: this.state,
});
console.log("Fetching battle list...");
const battleList = await getBattleList(this.state);
const allProgress: Record<string, Progress> = {};
const redraw = (name: string, progress: Progress) => {
allProgress[name] = progress;
@ -266,17 +340,39 @@ export class App {
})),
);
};
const endBar = () => {
bar?.end();
};
return { redraw, endBar };
}
async _exportOnce() {
const exporters = await this.getExporters();
const skipMode = this.getSkipMode();
const stats: Record<string, number> = Object.fromEntries(
exporters.map((e) => [e.name, 0]),
);
if (skipMode.includes("vs")) {
console.log("Skip exporting VS games.");
} else {
console.log("Fetching battle list...");
const gameList = await getBattleList(this.state);
const { redraw, endBar } = this.exporterProgress("Export games");
const fetcher = new GameFetcher({
cache: this.opts.cache ?? new FileCache(this.state.cacheDir),
state: this.state,
});
await Promise.all(
exporters.map((e) =>
showError(
this.exportBattleList({
this.exportGameList({
type: "VsInfo",
fetcher,
exporter: e,
battleList,
gameList,
onStep: (progress) => redraw(e.name, progress),
})
.then((count) => {
@ -289,7 +385,47 @@ export class App {
),
);
bar?.end();
endBar();
}
if (skipMode.includes("coop")) {
console.log("Skip exporting Coop games.");
} else {
console.log("Fetching coop battle list...");
const coopBattleList = await getBattleList(
this.state,
BattleListType.Coop,
);
const { redraw, endBar } = this.exporterProgress("Export games");
const fetcher = new GameFetcher({
cache: this.opts.cache ?? new FileCache(this.state.cacheDir),
state: this.state,
});
await Promise.all(
// TODO: remove this filter when stat.ink support coop export
exporters.filter((e) => e.name !== "stat.ink").map((e) =>
showError(
this.exportGameList({
type: "CoopInfo",
fetcher,
exporter: e,
gameList: coopBattleList,
onStep: (progress) => redraw(e.name, progress),
})
.then((count) => {
stats[e.name] = count;
}),
)
.catch((err) => {
console.error(`\nFailed to export to ${e.name}:`, err);
})
),
);
endBar();
}
console.log(
`Exported ${
@ -298,9 +434,6 @@ export class App {
.join(", ")
}`,
);
} finally {
bar?.end();
}
}
async monitor() {
while (true) {
@ -376,26 +509,26 @@ export class App {
}
}
/**
* Export battle list.
* Export game list.
*
* @param fetcher BattleFetcher
* @param exporter BattleExporter
* @param battleList ID list of battles, sorted by date, newest first
* @param onStep Callback function called when a battle is exported
* @param gameList ID list of games, sorted by date, newest first
* @param onStep Callback function called when a game is exported
*/
async exportBattleList(
{
async exportGameList({
type,
fetcher,
exporter,
battleList,
gameList,
onStep,
}: {
fetcher: BattleFetcher;
exporter: BattleExporter<VsBattle>;
battleList: string[];
onStep?: (progress: Progress) => void;
},
): Promise<number> {
type: Game["type"];
exporter: GameExporter;
fetcher: GameFetcher;
gameList: string[];
onStep: (progress: Progress) => void;
}) {
let exported = 0;
onStep?.({
@ -403,11 +536,17 @@ export class App {
total: 1,
});
const workQueue = [...await exporter.notExported(battleList)].reverse();
const workQueue = [
...await exporter.notExported({
type,
list: gameList,
}),
]
.reverse();
const step = async (battle: string) => {
const detail = await fetcher.fetchBattle(battle);
await exporter.exportBattle(detail);
const step = async (id: string) => {
const detail = await fetcher.fetch(type, id);
await exporter.exportGame(detail);
exported += 1;
onStep?.({
current: exported,

View File

@ -1,7 +1,7 @@
import type { StatInkPostBody, VsHistoryDetail } from "./types.ts";
export const AGENT_NAME = "s3si.ts";
export const S3SI_VERSION = "0.1.7";
export const S3SI_VERSION = "0.1.8";
export const NSOAPP_VERSION = "2.3.1";
export const WEB_VIEW_VERSION = "1.0.0-216d0219";
export const S3SI_LINK = "https://github.com/spacemeowx2/s3si.ts"

View File

@ -1,14 +1,14 @@
import { BattleExporter, VsBattle } from "../types.ts";
import { CoopInfo, GameExporter, VsInfo } from "../types.ts";
import { path } from "../../deps.ts";
import { NSOAPP_VERSION, S3SI_VERSION } from "../constant.ts";
import { parseVsHistoryDetailId } from "../utils.ts";
import { parseHistoryDetailId } from "../utils.ts";
export type FileExporterType = {
type: "VS" | "COOP";
nsoVersion: string;
s3siVersion: string;
exportTime: string;
data: VsBattle;
data: VsInfo | CoopInfo;
};
/**
@ -26,27 +26,27 @@ function replacer(key: string, value: unknown): unknown {
* This is useful for debugging. It will write each battle detail to a file.
* Timestamp is used as filename. Example: 20210101T000000Z.json
*/
export class FileExporter implements BattleExporter<VsBattle> {
export class FileExporter implements GameExporter {
name = "file";
constructor(private exportPath: string) {
}
getFilenameById(id: string) {
const { uid, timestamp } = parseVsHistoryDetailId(id);
const { uid, timestamp } = parseHistoryDetailId(id);
return `${uid}_${timestamp}Z.json`;
}
async exportBattle(battle: VsBattle) {
async exportGame(info: VsInfo | CoopInfo) {
await Deno.mkdir(this.exportPath, { recursive: true });
const filename = this.getFilenameById(battle.detail.id);
const filename = this.getFilenameById(info.detail.id);
const filepath = path.join(this.exportPath, filename);
const body: FileExporterType = {
type: "VS",
type: info.type === "VsInfo" ? "VS" : "COOP",
nsoVersion: NSOAPP_VERSION,
s3siVersion: S3SI_VERSION,
exportTime: new Date().toISOString(),
data: battle,
data: info,
};
await Deno.writeTextFile(
@ -54,7 +54,7 @@ export class FileExporter implements BattleExporter<VsBattle> {
JSON.stringify(body, replacer),
);
}
async notExported(list: string[]): Promise<string[]> {
async notExported({ list }: { list: string[] }): Promise<string[]> {
const out: string[] = [];
for (const id of list) {

View File

@ -5,17 +5,18 @@ import {
USERAGENT,
} from "../constant.ts";
import {
BattleExporter,
CoopInfo,
GameExporter,
StatInkPlayer,
StatInkPostBody,
StatInkStage,
VsBattle,
VsHistoryDetail,
VsInfo,
VsPlayer,
} from "../types.ts";
import { base64, msgpack } from "../../deps.ts";
import { APIError } from "../APIError.ts";
import { battleId, cache } from "../utils.ts";
import { cache, gameId } from "../utils.ts";
const S3S_NAMESPACE = "b3a2dbf5-2c09-4792-b78c-00b548b70aeb";
@ -40,7 +41,7 @@ const getStage = cache(_getStage);
*
* This is the default exporter. It will upload each battle detail to stat.ink.
*/
export class StatInkExporter implements BattleExporter<VsBattle> {
export class StatInkExporter implements GameExporter {
name = "stat.ink";
constructor(private statInkApiKey: string, private uploadMode: string) {
@ -54,8 +55,12 @@ export class StatInkExporter implements BattleExporter<VsBattle> {
"Authorization": `Bearer ${this.statInkApiKey}`,
};
}
async exportBattle(battle: VsBattle) {
const body = await this.mapBattle(battle);
async exportGame(game: VsInfo | CoopInfo) {
if (game.type === "CoopInfo") {
// TODO: support coop
return;
}
const body = await this.mapBattle(game);
const resp = await fetch("https://stat.ink/api/v3/battle", {
method: "POST",
@ -86,7 +91,7 @@ export class StatInkExporter implements BattleExporter<VsBattle> {
});
}
}
async notExported(list: string[]): Promise<string[]> {
async notExported({ list }: { list: string[] }): Promise<string[]> {
const uuid = await (await fetch("https://stat.ink/api/v3/s3s/uuid-list", {
headers: this.requestHeaders(),
})).json();
@ -94,8 +99,8 @@ export class StatInkExporter implements BattleExporter<VsBattle> {
const out: string[] = [];
for (const id of list) {
const s3sId = await battleId(id, S3S_NAMESPACE);
const s3siId = await battleId(id);
const s3sId = await gameId(id, S3S_NAMESPACE);
const s3siId = await gameId(id);
if (!uuid.includes(s3sId) && !uuid.includes(s3siId)) {
out.push(id);
@ -166,7 +171,7 @@ export class StatInkExporter implements BattleExporter<VsBattle> {
}
async mapBattle(
{ challengeProgress, bankaraMatchChallenge, listNode, detail: vsDetail }:
VsBattle,
VsInfo,
): Promise<StatInkPostBody> {
const {
knockout,
@ -185,7 +190,7 @@ export class StatInkExporter implements BattleExporter<VsBattle> {
const startedAt = Math.floor(new Date(playedTime).getTime() / 1000);
const result: StatInkPostBody = {
uuid: await battleId(vsDetail.id),
uuid: await gameId(vsDetail.id),
lobby: this.mapLobby(vsDetail),
rule: SPLATNET3_STATINK_MAP.RULE[vsDetail.vsRule.rule],
stage: await this.mapStage(vsDetail),

View File

@ -13,7 +13,7 @@ import {
RespMap,
VarsMap,
} from "./types.ts";
import { battleId } from "./utils.ts";
import { gameId } from "./utils.ts";
async function request<Q extends Queries>(
state: State,
@ -90,7 +90,9 @@ export async function checkToken(state: State) {
}
}
function getIdsFromGroups({ historyGroups }: { historyGroups: HistoryGroups }) {
function getIdsFromGroups<T extends { id: string }>(
{ historyGroups }: { historyGroups: HistoryGroups<T> },
) {
return historyGroups.nodes.flatMap((i) => i.historyDetails.nodes).map((i) =>
i.id
);
@ -112,6 +114,9 @@ const BATTLE_LIST_TYPE_MAP: Record<
[BattleListType.Private]: (state: State) =>
request(state, Queries.PrivateBattleHistoriesQuery)
.then((r) => getIdsFromGroups(r.privateBattleHistories)),
[BattleListType.Coop]: (state: State) =>
request(state, Queries.CoopHistoryQuery)
.then((r) => getIdsFromGroups(r.coopResult)),
};
export async function getBattleList(
@ -134,12 +139,31 @@ export function getBattleDetail(
);
}
export function getCoopDetail(
state: State,
id: string,
) {
return request(
state,
Queries.CoopHistoryDetailQuery,
{
coopHistoryDetailId: id,
},
);
}
export async function getBankaraBattleHistories(state: State) {
const resp = await request(state, Queries.BankaraBattleHistoriesQuery);
for (const i of resp.bankaraBattleHistories.historyGroups.nodes) {
for (const j of i.historyDetails.nodes) {
j._bid = await battleId(j.id);
j._bid = await gameId(j.id);
}
}
return resp;
}
export async function getCoopHistories(state: State) {
const resp = await request(state, Queries.CoopHistoryQuery);
return resp;
}

View File

@ -46,11 +46,15 @@ export type BattleListNode = {
udemae: string;
judgement: "LOSE" | "WIN" | "DEEMED_LOSE" | "EXEMPTED_LOSE";
};
export type HistoryGroups = {
export type CoopListNode = {
id: string;
};
export type HistoryGroups<T> = {
nodes: {
bankaraMatchChallenge: null | BankaraMatchChallenge;
historyDetails: {
nodes: BattleListNode[];
nodes: T[];
};
}[];
};
@ -96,13 +100,20 @@ export type ChallengeProgress = {
loseCount: number;
};
// With challenge info
export type VsBattle = {
type: "VsBattle";
export type VsInfo = {
type: "VsInfo";
listNode: null | BattleListNode;
bankaraMatchChallenge: null | BankaraMatchChallenge;
challengeProgress: null | ChallengeProgress;
detail: VsHistoryDetail;
};
// Salmon run
export type CoopInfo = {
type: "CoopInfo";
listNode: null | CoopListNode;
detail: CoopHistoryDetail;
};
export type Game = VsInfo | CoopInfo;
export type VsHistoryDetail = {
id: string;
vsRule: {
@ -138,16 +149,21 @@ export type VsHistoryDetail = {
awards: { name: string; rank: string }[];
duration: number;
};
export type CoopHistoryDetail = {
id: string;
};
export type BattleExporter<
D extends {
// type is seful when you implement more than one BattleExporter on the same class
export type GameExporter<
T extends {
// type is seful when you implement more than one GameExporter on the same class
type: string;
},
} = Game,
> = {
name: string;
notExported: (list: string[]) => Promise<string[]>;
exportBattle: (detail: D) => Promise<void>;
notExported: (
{ type, list }: { type: T["type"]; list: string[] },
) => Promise<string[]>;
exportGame: (game: T) => Promise<void>;
};
export type RespMap = {
@ -171,29 +187,35 @@ export type RespMap = {
};
[Queries.LatestBattleHistoriesQuery]: {
latestBattleHistories: {
historyGroups: HistoryGroups;
historyGroups: HistoryGroups<BattleListNode>;
};
};
[Queries.RegularBattleHistoriesQuery]: {
regularBattleHistories: {
historyGroups: HistoryGroups;
historyGroups: HistoryGroups<BattleListNode>;
};
};
[Queries.BankaraBattleHistoriesQuery]: {
bankaraBattleHistories: {
historyGroups: HistoryGroups;
historyGroups: HistoryGroups<BattleListNode>;
};
};
[Queries.PrivateBattleHistoriesQuery]: {
privateBattleHistories: {
historyGroups: HistoryGroups;
historyGroups: HistoryGroups<BattleListNode>;
};
};
[Queries.VsHistoryDetailQuery]: {
vsHistoryDetail: VsHistoryDetail;
};
[Queries.CoopHistoryQuery]: Record<never, never>;
[Queries.CoopHistoryDetailQuery]: Record<never, never>;
[Queries.CoopHistoryQuery]: {
coopResult: {
historyGroups: HistoryGroups<CoopListNode>;
};
};
[Queries.CoopHistoryDetailQuery]: {
coopHistoryDetail: CoopHistoryDetail;
};
};
export type GraphQLResponse<T> = {
data: T;
@ -208,6 +230,7 @@ export enum BattleListType {
Regular,
Bankara,
Private,
Coop,
}
export type StatInkPlayer = {

View File

@ -83,11 +83,11 @@ export async function showError<T>(p: Promise<T>): Promise<T> {
}
/**
* @param id id of VeVsHistoryDetail
* @param id id of VsHistoryDetail or CoopHistoryDetail
* @param namespace uuid namespace
* @returns
*/
export function battleId(
export function gameId(
id: string,
namespace = S3SI_NAMESPACE,
): Promise<string> {
@ -96,22 +96,38 @@ export function battleId(
return uuid.v5.generate(namespace, tsUuid);
}
export function parseVsHistoryDetailId(id: string) {
/**
* @param id VsHistoryDetail id or CoopHistoryDetail id
*/
//CoopHistoryDetail-u-quoeuj7rhknjq3jkanmm:20221022T065633_25287bf9-d9a8-42b0-b070-e938da103547
export function parseHistoryDetailId(id: string) {
const plainText = new TextDecoder().decode(base64.decode(id));
const re = /VsHistoryDetail-([a-z0-9-]+):(\w+):(\d{8}T\d{6})_([0-9a-f-]{36})/;
if (!re.test(plainText)) {
throw new Error(`Invalid battle ID: ${plainText}`);
}
const [, uid, listType, timestamp, uuid] = plainText.match(re)!;
const vsRE =
/VsHistoryDetail-([a-z0-9-]+):(\w+):(\d{8}T\d{6})_([0-9a-f-]{36})/;
const coopRE = /CoopHistoryDetail-([a-z0-9-]+):(\d{8}T\d{6})_([0-9a-f-]{36})/;
if (vsRE.test(plainText)) {
const [, uid, listType, timestamp, uuid] = plainText.match(vsRE)!;
return {
type: "VsHistoryDetail",
uid,
listType,
timestamp,
uuid,
};
} else if (coopRE.test(plainText)) {
const [, uid, timestamp, uuid] = plainText.match(coopRE)!;
return {
type: "CoopHistoryDetail",
uid,
timestamp,
uuid,
};
} else {
throw new Error(`Invalid ID: ${plainText}`);
}
}
export const delay = (ms: number) =>