From 4ef7131750a3ca08a43853cf92683bee2e993037 Mon Sep 17 00:00:00 2001
From: Adam Kariv <akariv@users.noreply.github.com>
Date: Wed, 17 Oct 2018 21:45:31 +0300
Subject: [PATCH] Make sure that concatenate always outputs all fields (#38)

* Make sure that concatenate always outputs all fields

* v0.0.28
---
 dataflows/VERSION                   |  2 +-
 dataflows/processors/concatenate.py |  1 +
 tests/test_lib.py                   | 15 +++++++++++----
 3 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/dataflows/VERSION b/dataflows/VERSION
index 24ff855..1fe6958 100644
--- a/dataflows/VERSION
+++ b/dataflows/VERSION
@@ -1 +1 @@
-0.0.27
+0.0.28
diff --git a/dataflows/processors/concatenate.py b/dataflows/processors/concatenate.py
index fcf74f3..3a65dd8 100644
--- a/dataflows/processors/concatenate.py
+++ b/dataflows/processors/concatenate.py
@@ -101,6 +101,7 @@ def func(package):
         package.pkg.descriptor['resources'] = new_resources
         yield package.pkg
 
+        needed_fields = sorted(fields.keys())
         it = iter(package)
         for resource in it:
             if matcher.match(resource.res.name):
diff --git a/tests/test_lib.py b/tests/test_lib.py
index b3f001e..5e038d5 100644
--- a/tests/test_lib.py
+++ b/tests/test_lib.py
@@ -146,13 +146,20 @@ def test_concatenate():
             {'c': 6, 'd': 7},
         ],
         concatenate({
-            'f1': ['a', 'c'],
-            'f2': ['b', 'd']
+            'f1': ['a'],
+            'f2': ['b', 'c'],
+            'f3': ['d']
         })
     )
     results, _, _ = f.results()
-    d = [(r['f1'], r['f2']) for r in results[0]]
-    assert d == list(zip(range(1,7), range(2, 8)))
+    assert results[0] == [
+        {'f1': 1, 'f2': 2, 'f3': None},
+        {'f1': 2, 'f2': 3, 'f3': None},
+        {'f1': 3, 'f2': 4, 'f3': None},
+        {'f1': None, 'f2': 4, 'f3': 5},
+        {'f1': None, 'f2': 5, 'f3': 6},
+        {'f1': None, 'f2': 6, 'f3': 7}
+    ]
 
 
 def test_filter_rows():