Add test for concordence with lwt

Author: benjeffery

python/lwt_interface/dict_encoding_testlib.py

@@ -609,8 +609,8 @@ def test_index(self):
         del d["indexes"]
         lwt = lwt_module.LightweightTableCollection()
         lwt.fromdict(d)
-        # and a tc without indexes doesn't have the key
-        assert "indexes" not in lwt.asdict()
+        # and a tc without indexes has empty dict
+        assert lwt.asdict()["indexes"] == {}
 
         # Both columns must be provided, if one is
         for col in ("insertion", "removal"):

python/lwt_interface/tskit_lwt_interface.h

@@ -1594,11 +1594,6 @@ write_table_arrays(tsk_table_collection_t *tables, PyObject *dict)
     };
 
     for (j = 0; j < sizeof(table_descs) / sizeof(*table_descs); j++) {
-        if (strcmp(table_descs[j].name, "indexes") == 0
-            && !tsk_table_collection_has_index(tables, 0)) {
-            continue;
-        }
-
         table_dict = PyDict_New();
         if (table_dict == NULL) {
             goto out;

python/tests/test_tables.py

@@ -1326,10 +1326,7 @@ def test_index(self):
         index = tskit.TableCollectionIndexes()
         assert index.edge_insertion_order is None
         assert index.edge_removal_order is None
-        assert index.asdict() == {
-            "edge_insertion_order": None,
-            "edge_removal_order": None,
-        }
+        assert index.asdict() == {}
 
 
 class TestSortTables:
@@ -2265,7 +2262,7 @@ def test_asdict(self):
         t = ts.tables
         self.add_metadata(t)
         d1 = {
-            "encoding_version": (1, 1),
+            "encoding_version": (1, 2),
             "sequence_length": t.sequence_length,
             "metadata_schema": str(t.metadata_schema),
             "metadata": t.metadata_schema.encode_row(t.metadata),
@@ -2607,6 +2604,41 @@ def test_indexes_roundtrip(self, simple_ts_fixture):
         tables.drop_index()
         assert not tskit.TableCollection.fromdict(tables.asdict()).has_index()
 
+    def test_asdict_lwt_concordence(self, ts_fixture):
+        def check_concordence(d1, d2):
+            assert set(d1.keys()) == set(d2.keys())
+            for k1, v1 in d1.items():
+                v2 = d2[k1]
+                assert type(v1) == type(v2)
+                if type(v1) == dict:
+                    assert set(v1.keys()) == set(v2.keys())
+                    for sk1, sv1 in v1.items():
+                        sv2 = v2[sk1]
+                        assert type(sv1) == type(sv2)
+                        if type(sv1) == np.ndarray:
+                            assert np.array_equal(sv1, sv2) or (
+                                np.all(tskit.is_unknown_time(sv1))
+                                and np.all(tskit.is_unknown_time(sv2))
+                            )
+                        elif type(sv1) in [bytes, str]:
+                            assert sv1 == sv2
+                        else:
+                            raise AssertionError()
+
+                else:
+                    assert v1 == v2
+
+        tables = ts_fixture.dump_tables()
+        assert tables.has_index()
+        lwt = _tskit.LightweightTableCollection()
+        lwt.fromdict(tables.asdict())
+        check_concordence(lwt.asdict(), tables.asdict())
+
+        tables.drop_index()
+        lwt = _tskit.LightweightTableCollection()
+        lwt.fromdict(tables.asdict())
+        check_concordence(lwt.asdict(), tables.asdict())
+
 
 class TestEqualityOptions:
     def test_equals_provenance(self):

python/tskit/tables.py

@@ -146,7 +146,7 @@ class TableCollectionIndexes:
     edge_removal_order: np.ndarray = attr.ib(default=None)
 
     def asdict(self):
-        return attr.asdict(self)
+        return attr.asdict(self, filter=lambda k, v: v is not None)
 
 
 def keep_with_offset(keep, data, offset):
@@ -2081,7 +2081,7 @@ def provenances(self):
     @property
     def indexes(self):
         indexes = self._ll_tables.indexes
-        return TableCollectionIndexes(**indexes) if indexes is not None else None
+        return TableCollectionIndexes(**indexes)
 
     @indexes.setter
     def indexes(self, indexes):
@@ -2140,7 +2140,7 @@ def asdict(self):
         map of table names to the tables themselves was returned.
         """
         ret = {
-            "encoding_version": (1, 1),
+            "encoding_version": (1, 2),
             "sequence_length": self.sequence_length,
             "metadata_schema": str(self.metadata_schema),
             "metadata": self.metadata_schema.encode_row(self.metadata),
@@ -2152,9 +2152,8 @@ def asdict(self):
             "mutations": self.mutations.asdict(),
             "populations": self.populations.asdict(),
             "provenances": self.provenances.asdict(),
+            "indexes": self.indexes.asdict(),
         }
-        if self.indexes is not None:
-            ret["indexes"] = self.indexes.asdict()
         return ret
 
     @property