From d0ce31fb90371fd6e6869860a5e8ab8357437747 Mon Sep 17 00:00:00 2001 From: David Chen Date: Tue, 10 Nov 2015 10:48:33 +0800 Subject: [PATCH] Fixing date_value must be the date type. Compacting with luigi.hdfs.clients API upgraded --- luiti/luigi_extensions/task_base.py | 6 ++++++ luiti/manager/files.py | 4 ++-- luiti/utils/target_utils.py | 2 +- setup.py | 2 +- tests/test_manager.py | 4 ++-- tests/test_utils.py | 6 +++++- 6 files changed, 17 insertions(+), 7 deletions(-) diff --git a/luiti/luigi_extensions/task_base.py b/luiti/luigi_extensions/task_base.py index e35cbd8..1f3686a 100644 --- a/luiti/luigi_extensions/task_base.py +++ b/luiti/luigi_extensions/task_base.py @@ -39,6 +39,12 @@ def requires(self): run = NotImplementedError def __init__(self, *args, **kwargs): + # Fix date_value type + if "date_value" in kwargs: + kwargs["date_value"] = ArrowParameter.get(kwargs["date_value"]) + if len(args) == 1: # just the luiti's date_value parameter + args = (ArrowParameter.get(args[0]), ) + super(TaskBase, self).__init__(*args, **kwargs) TaskInit.setup(self) diff --git a/luiti/manager/files.py b/luiti/manager/files.py index fc8d40a..8ddbc9c 100644 --- a/luiti/manager/files.py +++ b/luiti/manager/files.py @@ -33,8 +33,8 @@ def soft_delete_files(*files): for file1 in sorted(files): print "[delete file]", file1 - if luigi.hdfs.client.exists(file1): - luigi.hdfs.client.rename(file1, file1 + delete_at_str) + if luigi.hdfs.clients.exists(file1): + luigi.hdfs.clients.rename(file1, file1 + delete_at_str) print else: print "[err] doesnt exist!" diff --git a/luiti/utils/target_utils.py b/luiti/utils/target_utils.py index 6670342..ffdd2c8 100644 --- a/luiti/utils/target_utils.py +++ b/luiti/utils/target_utils.py @@ -68,6 +68,6 @@ class HdfsClientClass(object): @cached_property def client(self): import luigi.hdfs - return luigi.hdfs.client + return luigi.hdfs.clients HdfsClient = HdfsClientClass() TargetUtils.HdfsClient = HdfsClient diff --git a/setup.py b/setup.py index 3a8dea1..1fe57a9 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ def get_static_files(root): platforms='any', install_requires=[ # 1. luigi related - "luigi == 1.1.2", + "luigi >=2.0,<2.2", "snakebite>=2.5,<2.6", "protobuf>=2.6,<2.7", "tornado>=4.0,<4.1", diff --git a/tests/test_manager.py b/tests/test_manager.py index c607cec..ac69a5d 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -145,8 +145,8 @@ def test_Table(self): from luiti.manager.lazy_data import ld self.assertTrue(len(Table.print_all_tasks(ld.result)[0]) > 6, """Example data is ([[1, 'ADay', 'project_A'], [2, 'BDay', 'project_A'], [3, 'CDay', 'project_A'], [4, 'DDay', 'project_A'], [5, 'FoobarDay', 'project_A'], [6, 'HDay', 'project_B'], [7, 'ImportPackagesDay', 'project_A'], [8, 'MultipleDependentDay', 'project_A'], ['total', 8, '']], ['', 'All Tasks', 'luiti_package'])""") - @mock.patch("luigi.hdfs.client.rename") - @mock.patch("luigi.hdfs.client.exists") + @mock.patch("luigi.hdfs.clients.rename") + @mock.patch("luigi.hdfs.clients.exists") def test_Files(self, exists, rename): from luiti.manager.files import Files diff --git a/tests/test_utils.py b/tests/test_utils.py index c63bb5c..9c48a03 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -111,6 +111,10 @@ def cached_property_1(self): self.assertEqual(fb1.property_1, "property_1") self.assertEqual(fb1.cached_property_1, "cached_property_1") + self.assertTrue(inspect.ismethod(Foobar.method_1)) + self.assertTrue(isinstance(Foobar.property_1, property)) + self.assertTrue(isinstance(Foobar.cached_property_1, cached_property), Foobar.cached_property_1) + Foobar.extend({ 'not_exist_str': "not_exist_str", 'method_1': lambda self: "method_2", @@ -126,7 +130,7 @@ def cached_property_1(self): self.assertTrue(isinstance(Foobar.not_exist_str, str)) self.assertTrue(inspect.ismethod(Foobar.method_1)) self.assertTrue(isinstance(Foobar.property_1, property)) - self.assertTrue(isinstance(Foobar.cached_property_1, cached_property)) + self.assertTrue(isinstance(Foobar.cached_property_1, cached_property), Foobar.cached_property_1) @mock.patch("luigi.hdfs.exists") @mock.patch("luigi.hdfs.remove")