[decompiler] new features + fixes, + other jak2 fixes

common/type_system/Type.cpp

@@ -913,6 +913,17 @@ bool StructureType::lookup_field(const std::string& name, Field* out) {
   return false;
 }
 
+void StructureType::override_field_type(const std::string& field_name, const TypeSpec& new_type) {
+  int i = 0;
+  for (auto& x : m_fields) {
+    if (x.name() == field_name) {
+      x.set_override_type(new_type);
+      m_overriden_fields.push_back(i);
+    }
+    ++i;
+  }
+}
+
 /////////////////
 // BasicType
 /////////////////
@@ -1035,7 +1046,9 @@ BitFieldType::BitFieldType(std::string parent, std::string name, int size, bool
 bool BitFieldType::lookup_field(const std::string& name, BitField* out) const {
   for (auto& field : m_fields) {
     if (field.name() == name) {
-      *out = field;
+      if (out) {
+        *out = field;
+      }
       return true;
     }
   }

common/type_system/Type.h

@@ -222,9 +222,14 @@ class Field {
   void set_dynamic();
   void set_array(int size);
   void set_inline();
+  void set_override_type(const TypeSpec& new_type) {
+    m_type = new_type;
+    m_override_type = true;
+  }
   void mark_as_user_placed() { m_placed_by_user = true; }
   std::string print() const;
   const TypeSpec& type() const { return m_type; }
+  const std::optional<TypeSpec> decomp_as_type() const { return m_decomp_as_ts; }
   TypeSpec& type() { return m_type; }
   bool is_inline() const { return m_inline; }
   bool is_array() const { return m_array; }
@@ -249,6 +254,7 @@ class Field {
 
   double field_score() const { return m_field_score; }
   void set_field_score(double value) { m_field_score = value; }
+  void set_decomp_as_ts(const TypeSpec& ts) { m_decomp_as_ts = ts; }
 
  private:
   friend class TypeSystem;
@@ -258,6 +264,7 @@ class Field {
 
   std::string m_name;
   TypeSpec m_type;
+  bool m_override_type = false;
   int m_offset = -1;
   bool m_inline =
       false;  // does not make sense if m_type is value, and not an array and not dynamic
@@ -269,6 +276,8 @@ class Field {
   bool m_placed_by_user = false;  // was this field placed manually by the user?
 
   double m_field_score = 0.;
+
+  std::optional<TypeSpec> m_decomp_as_ts = std::nullopt;
 };
 
 class StructureType : public ReferenceType {
@@ -282,6 +291,7 @@ class StructureType : public ReferenceType {
   std::string print() const override;
   void inherit(StructureType* parent);
   const std::vector<Field>& fields() const { return m_fields; }
+  const std::vector<int>& override_fields() const { return m_overriden_fields; }
   bool operator==(const Type& other) const override;
   std::string diff_impl(const Type& other) const override;
   std::string diff_structure_common(const StructureType& other) const;
@@ -302,6 +312,7 @@ class StructureType : public ReferenceType {
   void set_allow_misalign(bool misalign) { m_allow_misalign = misalign; }
   void set_gen_inspect(bool gen_inspect) { m_generate_inspect = gen_inspect; }
   int size() const { return m_size_in_mem; }
+  void override_field_type(const std::string& field_name, const TypeSpec& new_type);
 
  protected:
   friend class TypeSystem;
@@ -317,6 +328,7 @@ class StructureType : public ReferenceType {
   size_t first_unique_field_idx() const { return m_idx_of_first_unique_field; }
 
   std::vector<Field> m_fields;
+  std::vector<int> m_overriden_fields;
   bool m_dynamic = false;
   int m_size_in_mem = 0;
   bool m_pack = false;

common/type_system/TypeSystem.cpp

@@ -951,7 +951,8 @@ int TypeSystem::add_field_to_type(StructureType* type,
                                   int array_size,
                                   int offset_override,
                                   bool skip_in_static_decomp,
-                                  double score) {
+                                  double score,
+                                  const std::optional<TypeSpec> decomp_as_ts) {
   if (type->lookup_field(field_name, nullptr)) {
     throw_typesystem_error("Type {} already has a field named {}\n", type->get_name(), field_name);
   }
@@ -993,6 +994,9 @@ int TypeSystem::add_field_to_type(StructureType* type,
     field.set_skip_in_static_decomp();
   }
   field.set_field_score(score);
+  if (decomp_as_ts) {
+    field.set_decomp_as_ts(*decomp_as_ts);
+  }
 
   int after_field = offset + get_size_in_type(field);
   if (type->get_size_in_memory() < after_field) {
@@ -2006,9 +2010,18 @@ std::string TypeSystem::generate_deftype_for_structure(const StructureType* st)
 
   const std::string inline_string = ":inline";
   const std::string dynamic_string = ":dynamic";
-  const std::string user_offset_string = ":offset xxx";
   bool has_offset_assert = false;
 
+  // calculate longest strings needed, for basic linting
+
+  // override fields
+  for (auto i : st->override_fields()) {
+    const auto& field = st->fields().at(i);
+    longest_field_name = std::max(longest_field_name, int(field.name().size()));
+    longest_type_name = std::max(longest_type_name, int(field.type().print().size()));
+  }
+
+  // normal fields
   for (size_t i = st->first_unique_field_idx(); i < st->fields().size(); i++) {
     const auto& field = st->fields().at(i);
     longest_field_name = std::max(longest_field_name, int(field.name().size()));
@@ -2040,6 +2053,21 @@ std::string TypeSystem::generate_deftype_for_structure(const StructureType* st)
     longest_mods = std::max(longest_mods, mods);
   }
 
+  // now actually write out the fields
+
+  // override fields first
+  for (auto i : st->override_fields()) {
+    const auto& field = st->fields().at(i);
+    result += "(";
+    result += field.name();
+    result.append(1 + (longest_field_name - int(field.name().size())), ' ');
+    result += field.type().print();
+    result.append(1 + (longest_type_name - int(field.type().print().size())), ' ');
+    result.append(1 + longest_mods, ' ');
+    result.append(":override)\n   ");
+  }
+
+  // now normal fields
   for (size_t i = st->first_unique_field_idx(); i < st->fields().size(); i++) {
     const auto& field = st->fields().at(i);
     result += "(";

common/type_system/TypeSystem.h

@@ -216,7 +216,8 @@ class TypeSystem {
                         int array_size = -1,
                         int offset_override = -1,
                         bool skip_in_static_decomp = false,
-                        double score = 0.0);
+                        double score = 0.0,
+                        const std::optional<TypeSpec> decomp_as_ts = std::nullopt);
 
   void add_builtin_types(GameVersion version);
 

common/type_system/deftype.cpp

@@ -103,6 +103,9 @@ void add_field(StructureType* structure,
   int offset_assert = -1;
   double score = 0;
   bool skip_in_decomp = false;
+  std::optional<TypeSpec> decomp_as_ts = std::nullopt;
+  Field override_field;
+  bool override = false;
 
   if (!rest->is_empty_list()) {
     if (car(rest).is_int()) {
@@ -125,6 +128,12 @@ void add_field(StructureType* structure,
       } else if (opt_name == ":offset") {
         offset_override = get_int(car(rest));
         rest = cdr(rest);
+      } else if (opt_name == ":override") {
+        override = true;
+        if (!structure->lookup_field(name, &override_field)) {
+          throw std::runtime_error(
+              fmt::format("Field {} not found to override", name));
+        }
       } else if (opt_name == ":overlay-at") {
         auto field_name = symbol_string(car(rest));
         Field overlay_field;
@@ -137,6 +146,9 @@ void add_field(StructureType* structure,
       } else if (opt_name == ":score") {
         score = get_float(car(rest));
         rest = cdr(rest);
+      } else if (opt_name == ":decomp-as") {
+        decomp_as_ts = TypeSpec(symbol_string(car(rest)));
+        rest = cdr(rest);
       } else if (opt_name == ":offset-assert") {
         offset_assert = get_int(car(rest));
         if (offset_assert == -1) {
@@ -151,11 +163,29 @@ void add_field(StructureType* structure,
     }
   }
 
-  int actual_offset = ts->add_field_to_type(structure, name, type, is_inline, is_dynamic,
-                                            array_size, offset_override, skip_in_decomp, score);
-  if (offset_assert != -1 && actual_offset != offset_assert) {
-    throw std::runtime_error("Field " + name + " was placed at " + std::to_string(actual_offset) +
-                             " but offset-assert was set to " + std::to_string(offset_assert));
+  if (override && name == override_field.name()) {
+    // override field, e.g. (root collide-shape :overlay-at root)
+    // does not add new field, merely rewrites inherited one
+    if (!ts->tc(override_field.type(), type)) {
+      throw std::runtime_error(
+          fmt::format("Wanted to override field {}, but type {} isn't child of {}",
+                      override_field.name(), type.print(), override_field.type().print()));
+    }
+    if (override_field.is_inline() != is_inline || override_field.is_dynamic() != is_dynamic ||
+        (array_size != -1 && override_field.array_size() != array_size)) {
+      throw std::runtime_error(fmt::format("Wanted to override field {}, but some parameters were different",
+                      override_field.name()));
+    }
+    structure->override_field_type(override_field.name(), type);
+  } else {
+    // new unique field
+    int actual_offset =
+        ts->add_field_to_type(structure, name, type, is_inline, is_dynamic, array_size,
+                              offset_override, skip_in_decomp, score, decomp_as_ts);
+    if (offset_assert != -1 && actual_offset != offset_assert) {
+      throw std::runtime_error("Field " + name + " was placed at " + std::to_string(actual_offset) +
+                               " but offset-assert was set to " + std::to_string(offset_assert));
+    }
   }
 }
 
@@ -739,7 +769,7 @@ DeftypeResult parse_deftype(const goos::Object& deftype,
                       parent_type_name));
     }
     new_type->inherit(pto);
-    ts->forward_declare_type_as(name, "basic");
+    ts->forward_declare_type_as(name, pto->get_name());
     auto sr =
         parse_structure_def(new_type.get(), ts, field_list_obj, options_obj, constants_to_use);
     result.flags = sr.flags;
@@ -769,7 +799,7 @@ DeftypeResult parse_deftype(const goos::Object& deftype,
     auto pto = dynamic_cast<StructureType*>(ts->lookup_type(parent_type));
     ASSERT(pto);
     new_type->inherit(pto);
-    ts->forward_declare_type_as(name, "structure");
+    ts->forward_declare_type_as(name, pto->get_name());
     auto sr =
         parse_structure_def(new_type.get(), ts, field_list_obj, options_obj, constants_to_use);
     result.flags = sr.flags;

decompiler/ObjectFile/LinkedObjectFile.cpp

@@ -914,8 +914,25 @@ bool LinkedObjectFile::is_string(int seg, int byte_idx) const {
   if (type_tag_ptr < 0 || size_t(type_tag_ptr) >= words_by_seg.at(seg).size() * 4) {
     return false;
   }
-  auto& type_word = words_by_seg.at(seg).at(type_tag_ptr / 4);
-  return type_word.kind() == LinkedWord::TYPE_PTR && type_word.symbol_name() == "string";
+  int type_word_idx = type_tag_ptr / 4;
+  auto& type_word = words_by_seg.at(seg).at(type_word_idx);
+  if (type_word.kind() == LinkedWord::TYPE_PTR && type_word.symbol_name() == "string") {
+    // could be a string basic
+    // check if we're not right after a zero-length string array.
+    if (type_word_idx >= 3) {
+      auto& arr_type_word = words_by_seg.at(seg).at(type_word_idx - 3);
+      auto& arr_len_word = words_by_seg.at(seg).at(type_word_idx - 2);
+      auto& arr_alen_word = words_by_seg.at(seg).at(type_word_idx - 1);
+      if (arr_type_word.kind() == LinkedWord::TYPE_PTR && arr_type_word.symbol_name() == "array" &&
+          arr_len_word.kind() == LinkedWord::PLAIN_DATA && arr_len_word.data == 0 &&
+          arr_alen_word.kind() == LinkedWord::PLAIN_DATA && arr_alen_word.data == 0) {
+        return false;
+      }
+    }
+    // seems good.
+    return true;
+  }
+  return false;
 }
 
 /*!