-
Notifications
You must be signed in to change notification settings - Fork 56
/
build.py
58 lines (44 loc) · 1.86 KB
/
build.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
import logging
import sys
from logging import getLogger
from typing import Any, Optional
import luigi
from gokart.task import TaskOnKart
from gokart.utils import check_config, read_environ
class LoggerConfig:
def __init__(self, level: int):
self.logger = getLogger(__name__)
self.default_level = self.logger.level
self.level = level
def __enter__(self):
logging.disable(self.level - 10) # subtract 10 to disable below self.level
self.logger.setLevel(self.level)
return self
def __exit__(self, exception_type, exception_value, traceback):
logging.disable(self.default_level - 10) # subtract 10 to disable below self.level
self.logger.setLevel(self.default_level)
class GokartBuildError(Exception):
pass
def _get_output(task: TaskOnKart) -> Any:
output = task.output()
if isinstance(output, list) or isinstance(output, tuple):
return [t.load() for t in output]
if isinstance(output, dict):
return {k: t.load() for k, t in output.items()}
return output.load()
def _reset_register(keep={'gokart', 'luigi'}):
luigi.task_register.Register._reg = [x for x in luigi.task_register.Register._reg
if x.__module__.split('.')[0] in keep] # avoid TaskClassAmbigiousException
def build(task: TaskOnKart, return_value: bool = True, reset_register: bool = True, log_level: int = logging.ERROR) -> Optional[Any]:
"""
Run gokart task for local interpreter.
"""
if reset_register:
_reset_register()
read_environ()
check_config()
with LoggerConfig(level=log_level):
result = luigi.build([task], local_scheduler=True, detailed_summary=True)
if result.status == luigi.LuigiStatusCode.FAILED:
raise GokartBuildError(result.summary_text)
return _get_output(task) if return_value else None