feat: alpha, parsable garbage

.gitignore

@@ -3,5 +3,5 @@ node_modules
 .DS_Store
 cached
 debug.d.ts
-factorio.schema.d.ts
-factorio.schema.d.ts
+**/factorio.schema.d.ts
+**/.factorio-ts-cache

packages/json-schema-producer/src/from-website.ts

@@ -4,6 +4,7 @@ import { JSONSchema6 } from "json-schema";
 import { compile } from "json-schema-to-typescript";
 import { getHtml } from "./browser";
 import { classNames as globalClassNames } from "./globals";
+import { bigBadHacks } from "./hack";
 import { definitionTypes } from "./json-schema";
 import { scrapeClassPage } from "./scrape/classes";
 import { scrapeConcepts } from "./scrape/concepts";
@@ -76,7 +77,19 @@ export const produce = async ({
     additionalProperties: false,
   };
   await fs.writeFile("factorio.schema.json", JSON.stringify(schema));
-  const tso = await compile(schema as any, "FactorioApi", { format: false });
+  bigBadHacks.isReadingRefs = false;
+  const tso = await compile(schema as any, "FactorioApi", {
+    format: false,
+    $refOptions: {
+      resolve: {
+        external: false,
+        file: false,
+      } as any,
+      dereference: {
+        circular: "ignore",
+      },
+    },
+  });
   await fs.writeFile("factorio.schema.d.ts", tso);
 };
 

packages/json-schema-producer/src/hack.ts

@@ -0,0 +1,11 @@
+export const bigBadHacks = {
+  /**
+   * YIKES. The circular reference/deref logic in the json-schema-to-typescript
+   * module just falls apart perf-wise when processing refs. we have many, many
+   * circular refs. even if we ignore them and continue on, processing them just
+   * takes forever. really, we don't need to process them. we can just the `tsType`
+   * hack for refs, and inject our own type pointers :). so, go fast again, and
+   * HACK. sorry, world. im terrible
+   */
+  isReadingRefs: true,
+};

packages/json-schema-producer/src/json-schema.ts

@@ -1,4 +1,5 @@
 import type { JSONSchema6 } from "json-schema";
+import { bigBadHacks } from "./hack";
 import { unmodeled } from "./unmodeled-entities";
 
 export const definitionTypes: string[] = [];
@@ -66,6 +67,8 @@ export const fromLuaType = (ltype_: string): JSONSchema6 => {
     case "string":
     case "boolean":
       return { type: ltype };
+    case "nil":
+      return { type: "null" };
   }
   if (ltype.match(/defines\..+/)) {
     return {
@@ -79,8 +82,18 @@ export const fromLuaType = (ltype_: string): JSONSchema6 => {
     definitionTypes.push(ltype.trim());
     console.warn(`:::: Adding type ${ltype}`);
   }
-  const ref: JSONSchema6 = {
-    $ref: `#/definitions/${ltype}`,
-  };
+  const ref: JSONSchema6 = {};
+  Object.defineProperty(ref, "$ref", {
+    get() {
+      if (bigBadHacks.isReadingRefs) return `#/definitions/${ltype}`;
+      // return a def'n that has no children. such a weak defn has no children,
+      // and will create 0 circular refs
+      return "#/definitions/LocalisedString";
+    },
+  });
+  Object.defineProperty(ref, "tsType", {
+    value: ltype,
+    enumerable: false,
+  });
   return ref;
 };

packages/json-schema-producer/src/langs/typescript.ts

@@ -23,33 +23,65 @@ const toTsType = (jtype: JSONSchema6): string => {
   ) {
     return jtt;
   }
-  // abstract
+  // abstract`
   if (jtt === "array") {
     const items = jtype.items;
-    if (!Array.isArray(items)) throw new Error("invalid array");
-    return `${items.map((it) => toTsType(it as JSONSchema6)).join(" | ")}[]`;
+    if (Array.isArray(items)) {
+      throw new Error("unexpected array");
+    }
+    return `${toTsType(items as JSONSchema6)}[]`;
   }
   if (jtt === "object") {
     const props = jtype.properties as Record<string, JSONSchema6>;
-    if (!props) throw new Error("missing properties field on object");
-    const kvs = Object.entries(props).map(([k, v]) => `"${k}": ${toTsType(v)}`);
-    return `{ ${kvs.join(";")} }`;
+    const additionalProps = jtype.additionalProperties as Record<
+      string,
+      JSONSchema6
+    >;
+    if (!props && !additionalProps) {
+      throw new Error(
+        "missing properties/additionalProperties field on object"
+      );
+    }
+    const kvs = Object.entries(props || []).map(
+      ([k, v]) => `"${k}": ${toTsType(v)}`
+    );
+    const kvsStr = kvs.length ? `{ ${kvs.join(";") + ";"} }` : "";
+    const adpStr = additionalProps
+      ? `Record<string, ${toTsType(additionalProps)}>`
+      : "";
+
+    return [kvsStr, adpStr].filter(Boolean).join("&");
   }
   if (jtype.anyOf) {
     return `(${jtype.anyOf
       .map((t) => toTsType(t as JSONSchema6))
       .join(" | ")})`;
   }
-  throw new Error(`unknown type ${jtt}`);
+  if (typeof jtype.const === "string") return jtype.const;
+  if (typeof jtype.$ref === "string") {
+    const parts = jtype.$ref.split("/");
+    return parts[parts.length - 1];
+  }
+  throw new Error(`unknown type ${jtt} from ${JSON.stringify(jtype)}`);
 };
 
 export const withType = {
-  classProperty: (name: string, schema: JSONSchema6) => {
+  classProperty: (
+    name: string,
+    schema: JSONSchema6,
+    opts: { parseMode: boolean }
+  ) => {
     const mode = (schema.properties?.mode as JSONSchema6)?.const as string[];
-    if (!mode) throw new Error("property has no mode");
-    if (!Array.isArray(mode)) throw new Error("expected mode array");
-    const isRO = !mode.some((rw: string) => !!rw.match(/w/i));
-    return `${isRO ? "readonly " : ""}${name}: ${toTsType(
+    if (opts.parseMode) {
+      if (!mode) {
+        throw new Error("property has no mode");
+      }
+      if (!Array.isArray(mode)) throw new Error("expected mode array");
+    }
+    const isRO = opts.parseMode
+      ? !mode.some((rw: string) => !!rw.match(/w/i))
+      : false;
+    return `${isRO ? "readonly " : ""}"${name}": ${toTsType(
       schema.properties?.type as JSONSchema6
     )}`;
   },
@@ -66,19 +98,37 @@ export const withType = {
     if (!retSchema) {
       throw new Error("missing return value");
     }
-    const ret = toTsType(retSchema);
-    return `(${args}) => ${ret}`;
+    return `(${args}) => ${toTsType(retSchema)}`;
   },
-  class: (schema: JSONSchema6) => {
+  class: (schema: JSONSchema6, opts: { asStruct?: boolean }) => {
     const members = (schema.properties!.members as JSONSchema6)
       .properties as Record<string, JSONSchema6>;
-    const memberStrs = Object.entries(
+    const { properties: propNames, methods: methodNames } = Object.entries(
       members as Record<string, JSONSchema6>
-    ).map(([name, aSchema]) => {
-      if (aSchema.properties?.return)
-        return `${name}: ${withType.method(aSchema)}`;
-      return withType.classProperty(name, aSchema);
+    ).reduce(
+      (acc, [name, aSchema]) => {
+        if (aSchema.properties?.return) acc.methods.push(name);
+        else acc.properties.push(name);
+        return acc;
+      },
+      { properties: [] as string[], methods: [] as string[] }
+    );
+    const sort = (a: string, b: string) => a.localeCompare(b);
+    const propStrs = propNames.sort(sort).map((name) => {
+      return withType.classProperty(name, members[name], {
+        parseMode: !opts?.asStruct,
+      });
+    });
+    const methodStrs = methodNames.sort(sort).map((name) => {
+      return `${name}: ${withType.method(members[name])}`;
     });
-    return `\{\n${memberStrs.map((it) => `  ${it}`).join(";\n")}\n}`;
+    return `
+{
+  ${propStrs.length ? "/* properties */" : ""}
+  ${propStrs.map((it) => `  ${it};`).join("\n")}
+
+  ${methodStrs.length ? "/* methods */" : ""}
+  ${methodStrs.map((it) => `  ${it};`).join("\n")}
+}`;
   },
 };

packages/json-schema-producer/src/scrape/classes.ts

@@ -175,6 +175,8 @@ const parseMemberAttr = (_document: Document, sigEl: IElement) => {
     ".element-name",
     "failed to find member attr name element"
   ).textContent;
+  const paramEl = query(sigEl!, ".param-type");
+  const jsonSchemaAny: JSONSchema6 = { type: "any" };
   const schema: JSONSchema6 = {
     properties: {
       name: {
@@ -185,14 +187,10 @@ const parseMemberAttr = (_document: Document, sigEl: IElement) => {
       type: [/[a-zA-Z0-9_-]+ (\[\])?\S?\[/].some((m) =>
         sigEl.textContent.match(m)
       )
-        ? { type: "any" }
-        : fromLuaType(
-            query(
-              sigEl!,
-              ".param-type",
-              "failed to find member attr type element"
-            ).textContent.trim()
-          ),
+        ? paramEl
+          ? fromLuaType(paramEl.textContent)
+          : jsonSchemaAny
+        : jsonSchemaAny,
       mode: {
         const:
           query(sigEl!, ".attribute-mode")
@@ -283,7 +281,10 @@ const scrapeClassFromEl = (
       },
     },
   };
-  schema.tsType = withType.class(schema);
+  Object.defineProperty(schema, "tsType", {
+    value: withType.class(schema, { asStruct: false }),
+    enumerable: false,
+  });
   return schema;
 };
 

packages/json-schema-producer/src/scrape/concepts.ts

@@ -1,6 +1,7 @@
 import { IDocument, IElement } from "happy-dom";
 import { JSONSchema6 } from "json-schema";
 import { query } from "../dom-extensions";
+import { withType } from "../langs/typescript";
 import { asMarkdown } from "../markdown";
 import { parseParam } from "./classes";
 
@@ -26,11 +27,28 @@ export const scrapeConcept = (el: IElement) => {
         const: name,
       },
       members: {
-        type: "array",
-        items: fields,
+        type: "object",
+        properties: fields.reduce((acc, field) => {
+          const memberName = (field.properties as any)?.name?.const;
+          if (typeof memberName !== "string") {
+            // @todo, actually parse these child structs
+            // hack! dirty workaround for inconsistent dom structure in Concepts.html
+            console.warn(
+              `dropping concept ${name} member: ${JSON.stringify(field)}`
+            );
+            return acc;
+          }
+          acc[memberName] = field;
+          return acc;
+        }, {} as Record<string, JSONSchema6>),
       },
     },
   };
+  Object.defineProperty(schema, "tsType", {
+    enumerable: false,
+    value: withType.class(schema, { asStruct: true }),
+  });
+
   return schema;
 };
 

packages/json-schema-producer/src/unmodeled-entities.ts

@@ -24,4 +24,12 @@ export const unmodeled = [
   "Depending on type",
   "peak",
   "restriction",
+  "CustomArray of Alignment",
+  "Icon",
+  "Connection",
+  "TabAndContent",
+  "MapGenPreset",
+  "UnitSpawnDefinition",
+  "CircuitConnectionDefinition",
+  "CraftingQueueItem",
 ];