Fixing date_value must be the date type. Compacting with luigi.hdfs.c…

…lients API upgraded

luiti/luigi_extensions/task_base.py

@@ -39,6 +39,12 @@ def requires(self):
     run = NotImplementedError
 
     def __init__(self, *args, **kwargs):
+        # Fix date_value type
+        if "date_value" in kwargs:
+            kwargs["date_value"] = ArrowParameter.get(kwargs["date_value"])
+        if len(args) == 1:  # just the luiti's date_value parameter
+            args = (ArrowParameter.get(args[0]), )
+
         super(TaskBase, self).__init__(*args, **kwargs)
         TaskInit.setup(self)
 

luiti/manager/files.py

@@ -33,8 +33,8 @@ def soft_delete_files(*files):
 
         for file1 in sorted(files):
             print "[delete file]", file1
-            if luigi.hdfs.client.exists(file1):
-                luigi.hdfs.client.rename(file1, file1 + delete_at_str)
+            if luigi.hdfs.clients.exists(file1):
+                luigi.hdfs.clients.rename(file1, file1 + delete_at_str)
                 print
             else:
                 print "[err] doesnt exist!"

luiti/utils/target_utils.py

@@ -68,6 +68,6 @@ class HdfsClientClass(object):
     @cached_property
     def client(self):
         import luigi.hdfs
-        return luigi.hdfs.client
+        return luigi.hdfs.clients
 HdfsClient = HdfsClientClass()
 TargetUtils.HdfsClient = HdfsClient

setup.py

@@ -52,7 +52,7 @@ def get_static_files(root):
     platforms='any',
     install_requires=[
         # 1. luigi related
-        "luigi         == 1.1.2",
+        "luigi         >=2.0,<2.2",
         "snakebite>=2.5,<2.6",
         "protobuf>=2.6,<2.7",
         "tornado>=4.0,<4.1",

tests/test_manager.py

@@ -145,8 +145,8 @@ def test_Table(self):
         from luiti.manager.lazy_data import ld
         self.assertTrue(len(Table.print_all_tasks(ld.result)[0]) > 6, """Example data is ([[1, 'ADay', 'project_A'], [2, 'BDay', 'project_A'], [3, 'CDay', 'project_A'], [4, 'DDay', 'project_A'], [5, 'FoobarDay', 'project_A'], [6, 'HDay', 'project_B'], [7, 'ImportPackagesDay', 'project_A'], [8, 'MultipleDependentDay', 'project_A'], ['total', 8, '']], ['', 'All Tasks', 'luiti_package'])""")
 
-    @mock.patch("luigi.hdfs.client.rename")
-    @mock.patch("luigi.hdfs.client.exists")
+    @mock.patch("luigi.hdfs.clients.rename")
+    @mock.patch("luigi.hdfs.clients.exists")
     def test_Files(self, exists, rename):
         from luiti.manager.files import Files
 

tests/test_utils.py

@@ -111,6 +111,10 @@ def cached_property_1(self):
         self.assertEqual(fb1.property_1, "property_1")
         self.assertEqual(fb1.cached_property_1, "cached_property_1")
 
+        self.assertTrue(inspect.ismethod(Foobar.method_1))
+        self.assertTrue(isinstance(Foobar.property_1, property))
+        self.assertTrue(isinstance(Foobar.cached_property_1, cached_property), Foobar.cached_property_1)
+
         Foobar.extend({
             'not_exist_str': "not_exist_str",
             'method_1': lambda self: "method_2",
@@ -126,7 +130,7 @@ def cached_property_1(self):
         self.assertTrue(isinstance(Foobar.not_exist_str, str))
         self.assertTrue(inspect.ismethod(Foobar.method_1))
         self.assertTrue(isinstance(Foobar.property_1, property))
-        self.assertTrue(isinstance(Foobar.cached_property_1, cached_property))
+        self.assertTrue(isinstance(Foobar.cached_property_1, cached_property), Foobar.cached_property_1)
 
     @mock.patch("luigi.hdfs.exists")
     @mock.patch("luigi.hdfs.remove")