From 0cc0f6033b7ccc85be44140d2c811dadaca25d9d Mon Sep 17 00:00:00 2001 From: Noah Manneschmidt Date: Mon, 16 Oct 2023 21:04:16 -0700 Subject: [PATCH] First steps at in-browser parsing support --- .github/workflows/build.yml | 2 +- .tool-versions | 2 +- .vscode/settings.json | 3 + package.json | 8 +- src/browser.ts | 1 - src/browser/index.ts | 123 +++++++++++++++++++++ src/browser/parseSong.ts | 194 ++++++++++++++++++++++++++++++++++ src/browser/shared.ts | 46 ++++++++ src/parseSong.ts | 32 +----- src/parsers/index.ts | 12 +++ src/{ => parsers}/parseDwi.ts | 25 +---- src/{ => parsers}/parseSm.ts | 8 +- src/{ => parsers}/parseSsc.ts | 8 +- src/parsers/types.ts | 19 ++++ src/util.ts | 6 +- tsconfig.json | 4 +- yarn.lock | 10 ++ 17 files changed, 438 insertions(+), 65 deletions(-) create mode 100644 .vscode/settings.json delete mode 100644 src/browser.ts create mode 100644 src/browser/index.ts create mode 100644 src/browser/parseSong.ts create mode 100644 src/browser/shared.ts create mode 100644 src/parsers/index.ts rename src/{ => parsers}/parseDwi.ts (95%) rename src/{ => parsers}/parseSm.ts (98%) rename src/{ => parsers}/parseSsc.ts (98%) create mode 100644 src/parsers/types.ts diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2204ba5..e69cf37 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,7 +20,7 @@ jobs: cache: "yarn" - name: Init project - run: yarn --frozen-lockfile + run: yarn --immutable - name: Webpack Build run: yarn build diff --git a/.tool-versions b/.tool-versions index a851188..3e51109 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1 @@ -nodejs 20.4.0 +nodejs 20 diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..25fa621 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "typescript.tsdk": "node_modules/typescript/lib" +} diff --git a/package.json b/package.json index c395203..93e529e 100644 --- a/package.json +++ b/package.json @@ -1,10 +1,10 @@ { "name": "simfile-parser", - "version": "0.6.1", + "version": "0.7.0", "description": "Read stepmania charts with javascript!", "type": "module", "main": "./dist/main.js", - "browser": "./dist/browser.js", + "browser": "./dist/browser/index.js", "types": "./dist/main.d.ts", "bin": "./dist/cli.js", "sideEffects": "false", @@ -57,6 +57,7 @@ "devDependencies": { "@types/jest": "^29.5.3", "@types/node": "^20.4.2", + "@types/wicg-file-system-access": "^2023.10.1", "@typescript-eslint/eslint-plugin": "^6.6.0", "@typescript-eslint/parser": "^6.6.0", "confusing-browser-globals": "^1.0.11", @@ -70,6 +71,9 @@ "ts-jest": "^29.1.1", "typescript": "^5.2.2" }, + "peerDependencies": { + "@types/wicg-file-system-access": "*" + }, "resolutions": { "semver": "^7.5.2" }, diff --git a/src/browser.ts b/src/browser.ts deleted file mode 100644 index 4d6334a..0000000 --- a/src/browser.ts +++ /dev/null @@ -1 +0,0 @@ -console.log("Sorry, no browser support yet"); diff --git a/src/browser/index.ts b/src/browser/index.ts new file mode 100644 index 0000000..94fbf09 --- /dev/null +++ b/src/browser/index.ts @@ -0,0 +1,123 @@ +import { Pack, Simfile } from "../types.js"; +import { reportError, printMaybeError } from "../util.js"; +import { parseSong } from "./parseSong.js"; +import { isDirectoryEntry, isDirectoryHandle } from "./shared.js"; + +/** + * @param dir directory handle + * @yields FileSystemDirectoryHandle for each subdir of the given dir + * @returns nothing + */ +async function* getDirectories( + dir: FileSystemDirectoryHandle | FileSystemDirectoryEntry +) { + if ("createReader" in dir) { + const dirs = await getDirectoriesFromEntry(dir); + yield* dirs; + } else { + for await (const child of dir.values()) { + if (child.kind === "directory") { + yield child; + } + } + } +} + +/** + * @param dir file system entry + * @returns only subdirectories of the given directory + */ +function getDirectoriesFromEntry(dir: FileSystemDirectoryEntry) { + const dirReader = dir.createReader(); + return new Promise((resolve, reject) => { + dirReader.readEntries((results) => { + resolve(results.filter(isDirectoryEntry)); + }, reject); + }); +} + +declare global { + interface DataTransferItem { + // optionalize this to avoid incorrect type narrowing below + getAsFileSystemHandle?(): Promise; + } +} + +/** + * Parse a pack drag/dropped by a user in a browser + * @param item a DataTransferItem from a drop event + * @returns parsed pack + */ +export async function parsePack(item: DataTransferItem | HTMLInputElement) { + let dir: FileSystemDirectoryEntry | FileSystemDirectoryHandle; + if (item instanceof HTMLInputElement) { + if ("webkitEntries" in item) { + const entries = item.webkitEntries; + if (entries.length !== 1) { + throw new Error("expected exactly one selected file"); + } + const entry = entries[0]; + if (!isDirectoryEntry(entry)) { + throw new Error("expected folder to be dropped, but got file"); + } + dir = entry; + } else { + throw new Error("entries property not available on provided input"); + } + } else { + if (item.kind !== "file") { + throw new Error("expected file to be dropped, but it was not a file"); + } + if (item.getAsFileSystemHandle) { + const dirHandle = await item.getAsFileSystemHandle(); + if (!dirHandle) { + throw new Error("could not get file handle from drop item"); + } + if (!isDirectoryHandle(dirHandle)) { + throw new Error("expected folder to be dropped, but got file"); + } + dir = dirHandle; + } else if ("webkitGetAsEntry" in item) { + const entry = item.webkitGetAsEntry(); + if (!entry) { + throw new Error("could not get a file entry from drop item"); + } + if (!isDirectoryEntry(entry)) { + throw new Error("expected folder to be dropped, but got file"); + } + dir = entry; + } else { + throw new Error("no supported file drop mechanism supported"); + } + } + + const pack: Pack = { + name: dir.name.replace(/-/g, " "), + dir: dir.name, + songCount: 0, + }; + + const simfiles: Simfile[] = []; + for await (const songFolder of getDirectories(dir)) { + try { + const songData = await parseSong(songFolder); + if (songData) { + simfiles.push({ + ...songData, + pack, + }); + } + } catch (e) { + reportError( + `parseStepchart failed for '${songFolder.name}': ${printMaybeError(e)}` + ); + } + } + + pack.songCount = simfiles.length; + + return { + ...pack, + simfiles, + }; +} diff --git a/src/browser/parseSong.ts b/src/browser/parseSong.ts new file mode 100644 index 0000000..398b3f0 --- /dev/null +++ b/src/browser/parseSong.ts @@ -0,0 +1,194 @@ +import { parsers, supportedExtensions } from "../parsers/index.js"; +import { ParsedImages, RawSimfile } from "../parsers/types.js"; +import { Simfile } from "../types.js"; +import { extname, isFileEntry } from "./shared.js"; + +/** + * Find a simfile in a given directory + * @param songDir directory handle + * @returns file handle for the song file + */ +async function getSongFile( + songDir: FileSystemDirectoryHandle | FileSystemDirectoryEntry +) { + if ("createReader" in songDir) { + return getSongFileFromEntry(songDir); + } + for await (const handle of songDir.values()) { + if (handle.kind === "file") { + if (supportedExtensions.some((ext) => handle.name.endsWith(ext))) { + return handle; + } + } + } + return null; +} + +/** + * @param songDir legacy file system entry + * @returns promise of the found file entry or null + */ +async function getSongFileFromEntry(songDir: FileSystemDirectoryEntry) { + const dirReader = songDir.createReader(); + return new Promise((resolve, reject) => { + dirReader.readEntries((results) => { + for (const result of results) { + if (isFileEntry(result)) { + if (supportedExtensions.some((ext) => result.name.endsWith(ext))) { + resolve(result); + return; + } + } + } + resolve(null); + }, reject); + }); +} + +const imageExts = new Set([".png", ".jpg"]); + +/** + * Get all image files in a given directory + * @param songDir directory + * @yields file handles filtered to supported image extentions + */ +async function* getImages( + songDir: FileSystemDirectoryHandle | FileSystemDirectoryEntry +) { + let files: + | AsyncIterable + | Iterable; + if ("values" in songDir) { + files = songDir.values(); + } else { + files = await new Promise((res, rej) => + songDir.createReader().readEntries(res, rej) + ); + } + for await (const file of files) { + const ext = extname(file.name); + if (!ext) { + continue; + } + if ("kind" in file && file.kind === "directory") { + continue; + } + if ("isFile" in file && !isFileEntry(file)) { + continue; + } + if (imageExts.has(ext)) { + yield file as FileSystemFileEntry | FileSystemFileHandle; + } + } +} + +/** + * Make some best guesses about which images should be used for which fields + * @param songDir path to a song directory + * @param tagged image metadata found in simfile + * @returns final image metadata + */ +async function guessImages( + songDir: FileSystemDirectoryHandle | FileSystemDirectoryEntry, + tagged: ParsedImages +) { + let jacket = tagged.jacket; + let bg = tagged.bg; + let banner = tagged.banner; + for await (const { name: imageName } of getImages(songDir)) { + const ext = extname(imageName)!; + if ( + (!jacket && imageName.endsWith("-jacket" + ext)) || + imageName.startsWith("jacket.") + ) { + jacket = imageName; + } + if ( + (!bg && imageName.endsWith("-bg" + ext)) || + imageName.startsWith("bg.") + ) { + bg = imageName; + } + if ( + (!banner && imageName.endsWith("-bn" + ext)) || + imageName.startsWith("bn.") + ) { + banner = imageName; + } + } + return { jacket, bg, banner }; +} + +/** + * get individual bpms of each chart + * @param sm simfile + * @returns list of found bpms, one per chart + */ +function getBpms(sm: Pick): number[] { + const chart = Object.values(sm.charts)[0]; + return chart.bpm.map((b) => b.bpm); +} + +/** + * Parse a single simfile. Automatically determines which parser to use depending on chart definition type. + * @param songDir path to song folder (contains a chart definition file [dwi/sm], images, etc) + * @returns a simfile object without mix info or null if no sm/ssc file was found + */ +export async function parseSong( + songDir: FileSystemDirectoryHandle | FileSystemDirectoryEntry +): Promise | null> { + const songFileHandleOrEntry = await getSongFile(songDir); + if (!songFileHandleOrEntry) { + return null; + } + const extension = extname(songFileHandleOrEntry.name); + if (!extension) return null; + + const parser = parsers[extension]; + + if (!parser) { + throw new Error(`No parser registered for extension: ${extension}`); + } + + let fileContents: File; + if ("getFile" in songFileHandleOrEntry) { + fileContents = await songFileHandleOrEntry.getFile(); + } else { + fileContents = await new Promise((resolve, reject) => + songFileHandleOrEntry.file(resolve, reject) + ); + } + + const { images, ...rawStepchart } = parser(await fileContents.text(), ""); + + if (!Object.keys(rawStepchart.charts).length) { + throw new Error( + `Failed to parse any charts from song: ${rawStepchart.title}` + ); + } + + const bpms = getBpms(rawStepchart); + const minBpm = Math.round(Math.min(...bpms)); + const maxBpm = Math.round(Math.max(...bpms)); + + let displayBpm = rawStepchart.displayBpm; + if (!displayBpm) { + displayBpm = minBpm === maxBpm ? minBpm.toString() : `${minBpm}-${maxBpm}`; + } + + const finalImages = await guessImages(songDir, images); + + return { + ...rawStepchart, + title: { + titleName: rawStepchart.title, + translitTitleName: rawStepchart.titletranslit ?? null, + titleDir: songDir.name, + ...finalImages, + }, + minBpm, + maxBpm, + displayBpm, + stopCount: Object.values(rawStepchart.charts)[0].stops.length, + }; +} diff --git a/src/browser/shared.ts b/src/browser/shared.ts new file mode 100644 index 0000000..5cfaeae --- /dev/null +++ b/src/browser/shared.ts @@ -0,0 +1,46 @@ +/** + * returns extension name from a filename + * @param name filename + * @returns extension, with leading period + */ +export function extname(name: string) { + const match = name.match(/.+(\.[^.]+)$/); + if (match) { + return match[1]; + } + return null; +} + +/** + * narrows the type of a file system entry + * @param handle file system entry + * @param handle.isFile anything + * @returns true if entry is a file + */ +export function isFileEntry(handle: { + isFile: boolean; +}): handle is FileSystemFileEntry { + return handle.isFile; +} + +/** + * narrows the type of a file system handle + * @param handle file system handle + * @returns true if handle is a dir + */ +export function isDirectoryHandle( + handle: FileSystemHandle +): handle is FileSystemDirectoryHandle { + return handle.kind === "directory"; +} + +/** + * narrows the type of a file system handle + * @param handle file system handle + * @returns true if handle is a dir + */ +export function isDirectoryEntry( + handle: FileSystemEntry +): handle is FileSystemDirectoryEntry { + return handle.isDirectory; +} diff --git a/src/parseSong.ts b/src/parseSong.ts index 77aa176..7e4c20c 100644 --- a/src/parseSong.ts +++ b/src/parseSong.ts @@ -1,31 +1,8 @@ import * as fs from "node:fs"; import * as path from "node:path"; -import { parseDwi } from "./parseDwi.js"; -import { parseSm } from "./parseSm.js"; -import { parseSsc } from "./parseSsc.js"; import { Simfile } from "./types.js"; - -export type RawSimfile = Omit< - Simfile, - "mix" | "title" | "minBpm" | "maxBpm" -> & { - title: string; - titletranslit: string | null; - displayBpm: string | undefined; - images: Images; -}; -export interface Images { - banner: string | null; - bg: string | null; - jacket: string | null; -} -type Parser = (simfileSource: string, titleDir: string) => RawSimfile; - -const parsers: Record = { - ".sm": parseSm, - ".ssc": parseSsc, - ".dwi": parseDwi, -}; +import { parsers, supportedExtensions } from "./parsers/index.js"; +import type { ParsedImages, RawSimfile } from "./parsers/types.js"; /** * Find a simfile in a given directory @@ -34,8 +11,7 @@ const parsers: Record = { */ function getSongFile(songDir: string) { const files = fs.readdirSync(songDir); - const extensions = Object.keys(parsers); - return files.find((f) => extensions.some((ext) => f.endsWith(ext))); + return files.find((f) => supportedExtensions.some((ext) => f.endsWith(ext))); } const imageExts = new Set([".png", ".jpg"]); @@ -55,7 +31,7 @@ function getImages(songDir: string): string[] { * @param tagged image metadata found in simfile * @returns final image metadata */ -function guessImages(songDir: string, tagged: Images) { +function guessImages(songDir: string, tagged: ParsedImages) { let jacket = tagged.jacket; let bg = tagged.bg; let banner = tagged.banner; diff --git a/src/parsers/index.ts b/src/parsers/index.ts new file mode 100644 index 0000000..84ab540 --- /dev/null +++ b/src/parsers/index.ts @@ -0,0 +1,12 @@ +import { parseDwi } from "./parseDwi.js"; +import { parseSm } from "./parseSm.js"; +import { parseSsc } from "./parseSsc.js"; +import { Parser } from "./types.js"; + +export const parsers: Record = { + ".sm": parseSm, + ".ssc": parseSsc, + ".dwi": parseDwi, +}; + +export const supportedExtensions = Object.keys(parsers); diff --git a/src/parseDwi.ts b/src/parsers/parseDwi.ts similarity index 95% rename from src/parseDwi.ts rename to src/parsers/parseDwi.ts index 3cf9976..c868f00 100644 --- a/src/parseDwi.ts +++ b/src/parsers/parseDwi.ts @@ -1,14 +1,13 @@ -import fs from "node:fs"; -import { Fraction } from "./fraction.js"; -import { RawSimfile } from "./parseSong.js"; +import { Fraction } from "../fraction.js"; +import { RawSimfile } from "./types.js"; import { determineBeat, mergeSimilarBpmRanges, normalizedDifficultyMap, printMaybeError, reportError, -} from "./util.js"; -import { Arrow, FreezeLocation, BpmChange } from "./types.js"; +} from "../util.js"; +import { Arrow, FreezeLocation, BpmChange } from "../types.js"; // eslint-disable-next-line jsdoc/require-jsdoc function isMetaTag(tag: string): tag is "title" | "artist" { @@ -252,20 +251,6 @@ function parseArrowStream( return { arrows, freezes }; } -/** - * @param titlePath path to song dir - * @returns filename of banner, if found - */ -function findBanner(titlePath: string): string | null { - const files = fs.readdirSync(titlePath); - - const bannerFile = files.find( - (f) => f.endsWith(".png") && !f.endsWith("-bg.png") - ); - - return bannerFile ?? null; -} - /** * parse a DWI file * @param dwi entire contents of the file @@ -287,7 +272,7 @@ export function parseDwi(dwi: string, titlePath?: string): RawSimfile { charts: {}, availableTypes: [], images: { - banner: titlePath ? findBanner(titlePath) : null, + banner: null, bg: null, jacket: null, }, diff --git a/src/parseSm.ts b/src/parsers/parseSm.ts similarity index 98% rename from src/parseSm.ts rename to src/parsers/parseSm.ts index 7cb1d68..fd53940 100644 --- a/src/parseSm.ts +++ b/src/parsers/parseSm.ts @@ -1,6 +1,6 @@ -import { Fraction } from "./fraction.js"; -import { RawSimfile } from "./parseSong.js"; -import { FreezeLocation, Arrow } from "./types.js"; +import { Fraction } from "../fraction.js"; +import { RawSimfile } from "./types.js"; +import { FreezeLocation, Arrow } from "../types.js"; import { determineBeat, mergeSimilarBpmRanges, @@ -8,7 +8,7 @@ import { printMaybeError, renameBackground, reportError, -} from "./util.js"; +} from "../util.js"; // Ref: https://github.com/stepmania/stepmania/wiki/sm diff --git a/src/parseSsc.ts b/src/parsers/parseSsc.ts similarity index 98% rename from src/parseSsc.ts rename to src/parsers/parseSsc.ts index 4f94445..8346d8f 100644 --- a/src/parseSsc.ts +++ b/src/parsers/parseSsc.ts @@ -1,12 +1,12 @@ -import { Fraction } from "./fraction.js"; -import { RawSimfile } from "./parseSong.js"; +import { Fraction } from "../fraction.js"; +import { RawSimfile } from "./types.js"; import { FreezeLocation, Arrow, StepchartType, Stepchart, Mode, -} from "./types.js"; +} from "../types.js"; import { determineBeat, mergeSimilarBpmRanges, @@ -14,7 +14,7 @@ import { printMaybeError, renameBackground, reportError, -} from "./util.js"; +} from "../util.js"; // Ref: https://github.com/stepmania/stepmania/wiki/ssc diff --git a/src/parsers/types.ts b/src/parsers/types.ts new file mode 100644 index 0000000..22924ee --- /dev/null +++ b/src/parsers/types.ts @@ -0,0 +1,19 @@ +import type { Simfile } from "../types.js"; + +export type RawSimfile = Omit< + Simfile, + "mix" | "title" | "minBpm" | "maxBpm" +> & { + title: string; + titletranslit: string | null; + displayBpm: string | undefined; + images: ParsedImages; +}; + +export interface ParsedImages { + banner: string | null; + bg: string | null; + jacket: string | null; +} + +export type Parser = (simfileSource: string, titleDir: string) => RawSimfile; diff --git a/src/util.ts b/src/util.ts index 4025fa7..c5028d9 100644 --- a/src/util.ts +++ b/src/util.ts @@ -1,5 +1,5 @@ import { Fraction } from "./fraction.js"; -import { Images } from "./parseSong.js"; +import { ParsedImages } from "./parsers/types.js"; import { Arrow, BpmChange, Difficulty } from "./types.js"; const beats = [ @@ -87,7 +87,9 @@ export function mergeSimilarBpmRanges(bpm: BpmChange[]): BpmChange[] { * if we found a `background` tag, rename it to `bg` * @param images image data */ -export function renameBackground(images: Images & { background?: string }) { +export function renameBackground( + images: ParsedImages & { background?: string } +) { if (typeof images.background !== "undefined") { if (images.background) { images.bg = images.background; diff --git a/tsconfig.json b/tsconfig.json index 2453e74..8647cfe 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -18,8 +18,8 @@ "allowSyntheticDefaultImports": true, "sourceMap": true, "outDir": "./dist/", - "types": ["node", "jest"], - "lib": ["ES2020"] + "types": ["node", "jest", "wicg-file-system-access"], + "lib": ["ES2020", "DOM"] }, "include": ["src/**/*.ts"], "exclude": ["node_modules", "**/*.test.ts"] diff --git a/yarn.lock b/yarn.lock index 6faa774..deac606 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1037,6 +1037,13 @@ __metadata: languageName: node linkType: hard +"@types/wicg-file-system-access@npm:^2023.10.1": + version: 2023.10.1 + resolution: "@types/wicg-file-system-access@npm:2023.10.1" + checksum: 2b91ec8657a1f773c2bca40e48a19f8f8e0ebb26b882dc82ea4814cc84b9d224872fd178aa6a7462c743f32d1971727314bc56313a88b516b0e40167d6dbc8b2 + languageName: node + linkType: hard + "@types/yargs-parser@npm:*": version: 21.0.0 resolution: "@types/yargs-parser@npm:21.0.0" @@ -4318,6 +4325,7 @@ __metadata: dependencies: "@types/jest": ^29.5.3 "@types/node": ^20.4.2 + "@types/wicg-file-system-access": ^2023.10.1 "@typescript-eslint/eslint-plugin": ^6.6.0 "@typescript-eslint/parser": ^6.6.0 confusing-browser-globals: ^1.0.11 @@ -4330,6 +4338,8 @@ __metadata: prettier: ^2.8.8 ts-jest: ^29.1.1 typescript: ^5.2.2 + peerDependencies: + "@types/wicg-file-system-access": "*" bin: simfile-parser: ./dist/cli.js languageName: unknown