🐛 hdf5 export fails for concat df with missing columns (#1493)

* test(core): hdf5 export fails for concat df with missing columns

* fix

Co-authored-by: Maarten A. Breddels <maartenbreddels@gmail.com>

packages/vaex-core/vaex/dataset.py

@@ -753,6 +753,9 @@ def _decode(cls, encoding, spec):
         return ds
 
     def is_masked(self, column):
+        for dataset in self.datasets:
+            if column not in dataset:
+                return True
         return any(k.is_masked(column) for k in self.datasets)
 
     def shape(self, column):

tests/concat_test.py

@@ -238,6 +238,8 @@ def test_concat_unaligned_schema(arrow):
     df = df1.concat(df2)
     assert df.x.tolist() == [1, 2, None, None]
     assert df.y.tolist() == [None, None, 'd', 'e']
+    assert df.is_masked('x')
+    assert df.is_masked('y')
     # always 'upcast' to Arrow arrays
     # # rationale: Arrow will use use less memory, numpy has no efficient way to represent all missing data
     assert df.x.data_type() == pa.float32()

tests/export_test.py

@@ -107,6 +107,20 @@ def test_export_open_hdf5(ds_local):
     ds_opened = vaex.open(filename)
     assert list(ds) == list(ds_opened)
 
+def test_export_concat_missing_cols_hdf5(tmpdir):
+    df1 = vaex.from_arrays(x=[1, 2, 3], s=['x1', 'x2', 'x3'], y=[10, 20, 30])
+    df2 = vaex.from_arrays(x=[4, 5, 6])
+
+    df = vaex.concat([df1, df2])
+
+    filename = tempfile.mktemp(suffix='.hdf5')
+    df.export_hdf5(filename)
+
+    df_opened = vaex.open(filename)
+    assert df_opened.x.tolist() == [1, 2, 3, 4, 5, 6]
+    assert df_opened.y.tolist() == [10, 20, 30, None, None, None]
+    assert df_opened.s.tolist() == ['x1', 'x2', 'x3', None, None, None]
+
 def test_export_open_csv(ds_local, tmpdir):
     df = ds_local
     path = str(tmpdir.join('test.csv'))