From cc5e00a1ad05189f925fb56c96fc5cbdc12e3184 Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 18 Jan 2019 08:30:50 +0000 Subject: [PATCH 01/38] :sparkles: initial prototype for any data loader --- moban/constants.py | 1 + moban/main.py | 3 +-- moban/plugins.py | 42 ++++++++++++++++++++++++++++++++------- moban/utils.py | 35 -------------------------------- tests/test_yaml_loader.py | 12 +++++------ 5 files changed, 43 insertions(+), 50 deletions(-) diff --git a/moban/constants.py b/moban/constants.py index b17eeaca..7f9cef62 100644 --- a/moban/constants.py +++ b/moban/constants.py @@ -93,6 +93,7 @@ JINJA_GLOBALS_EXTENSION = "jinja_globals" TEMPLATE_ENGINE_EXTENSION = "template_engine" +DATA_LOADER_EXTENSION = "data_loader" LIBRARY_EXTENSION = "library" MOBAN_EXTENSIONS = "^moban_.+$" diff --git a/moban/main.py b/moban/main.py index f811efa0..b9f2a65c 100644 --- a/moban/main.py +++ b/moban/main.py @@ -16,7 +16,6 @@ import moban.mobanfile as mobanfile import moban.exceptions as exceptions from moban import plugins -from moban.utils import merge, open_yaml from moban.hashstore import HASH_STORE @@ -122,7 +121,7 @@ def handle_moban_file(moban_file, options): """ act upon default moban file """ - moban_file_configurations = open_yaml(None, moban_file) + moban_file_configurations = plugins.load_data(None, moban_file) if moban_file_configurations is None: raise exceptions.MobanfileGrammarException( constants.ERROR_INVALID_MOBAN_FILE % moban_file diff --git a/moban/plugins.py b/moban/plugins.py index 13488cc5..0cdb94f0 100644 --- a/moban/plugins.py +++ b/moban/plugins.py @@ -164,8 +164,42 @@ def raise_exception(self, key): raise exceptions.NoThirdPartyEngine(key) +class AnyDataLoader(PluginManager): + def __init__(self): + super(AnyDataLoader, self).__init__( + constants.DATA_LOADER_EXTENSION + ) + + def get_data(self, file_name): + file_extension = os.path.splitext(file_name)[1] + file_type = file_extension + if file_extension.startswith('.'): + file_type = file_type[1:] + + loader_function = self.load_me_now(file_type) + return loader_function(file_name) + + LIBRARIES = LibraryManager() ENGINES = EngineFactory() +LOADERS = AnyDataLoader() + + +def load_data(base_dir, file_name): + abs_file_path = utils.search_file(base_dir, file_name) + data = LOADERS.get_data(abs_file_path) + if data is not None: + parent_data = None + if base_dir and constants.LABEL_OVERRIDES in data: + parent_data = load_data( + base_dir, data.pop(constants.LABEL_OVERRIDES) + ) + if parent_data: + return utils.merge(data, parent_data) + else: + return data + else: + return None def expand_template_directories(dirs): @@ -216,13 +250,7 @@ def __init__(self, context_dirs): def get_data(self, file_name): try: - file_extension = os.path.splitext(file_name)[1] - if file_extension == ".json": - data = utils.open_json(self.context_dirs, file_name) - elif file_extension in [".yml", ".yaml"]: - data = utils.open_yaml(self.context_dirs, file_name) - else: - raise exceptions.IncorrectDataInput + data = load_data(self.context_dirs, file_name) utils.merge(data, self.__cached_environ_variables) return data except (IOError, exceptions.IncorrectDataInput) as exception: diff --git a/moban/utils.py b/moban/utils.py index 36de191b..00b13774 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -1,12 +1,9 @@ import os import re import sys -import json import stat import errno -from ruamel.yaml import YAML - import moban.reporter as reporter import moban.constants as constants import moban.exceptions as exceptions @@ -32,38 +29,6 @@ def merge(left, right): return left -def open_yaml(base_dir, file_name): - """ - chained yaml loader - """ - the_yaml_file = search_file(base_dir, file_name) - with open(the_yaml_file, "r") as data_yaml: - yaml = YAML(typ="rt") - data = yaml.load(data_yaml) - if data is not None: - parent_data = None - if base_dir and constants.LABEL_OVERRIDES in data: - parent_data = open_yaml( - base_dir, data.pop(constants.LABEL_OVERRIDES) - ) - if parent_data: - return merge(data, parent_data) - else: - return data - else: - return None - - -def open_json(base_dir, file_name): - """ - returns json contents as string - """ - the_json_file = search_file(base_dir, file_name) - with open(the_json_file, "r") as json_data: - data = json.loads(json_data.read()) - return data - - def search_file(base_dir, file_name): the_file = file_name if not os.path.exists(the_file): diff --git a/tests/test_yaml_loader.py b/tests/test_yaml_loader.py index 40709c24..6c74abdf 100644 --- a/tests/test_yaml_loader.py +++ b/tests/test_yaml_loader.py @@ -2,34 +2,34 @@ from nose.tools import eq_, raises -from moban.utils import open_yaml +from moban.plugins import load_data def test_simple_yaml(): test_file = os.path.join("tests", "fixtures", "simple.yaml") - data = open_yaml(os.path.join("tests", "fixtures"), test_file) + data = load_data(os.path.join("tests", "fixtures"), test_file) eq_(data, {"simple": "yaml"}) def test_inheritance_yaml(): test_file = os.path.join("tests", "fixtures", "child.yaml") - data = open_yaml(os.path.join("tests", "fixtures", "config"), test_file) + data = load_data(os.path.join("tests", "fixtures", "config"), test_file) eq_(data, {"key": "hello world", "pass": "ox"}) @raises(IOError) def test_exception(): test_file = os.path.join("tests", "fixtures", "orphan.yaml") - open_yaml(os.path.join("tests", "fixtures", "config"), test_file) + load_data(os.path.join("tests", "fixtures", "config"), test_file) @raises(IOError) def test_exception_2(): test_file = os.path.join("tests", "fixtures", "dragon.yaml") - open_yaml(os.path.join("tests", "fixtures", "config"), test_file) + load_data(os.path.join("tests", "fixtures", "config"), test_file) @raises(IOError) def test_exception_3(): test_file = os.path.join("tests", "fixtures", "dragon.yaml") - open_yaml(None, test_file) + load_data(None, test_file) From dd4a4fd5732caca8ea33ba18a399dc95a0a9f08c Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 20 Jan 2019 13:09:51 +0000 Subject: [PATCH 02/38] :newspaper: add missing files --- moban/data_loader/__init__.py | 0 moban/data_loader/json.py | 15 +++++++++++++++ moban/data_loader/yaml.py | 12 ++++++++++++ moban/loaders.py | 8 ++++++++ 4 files changed, 35 insertions(+) create mode 100644 moban/data_loader/__init__.py create mode 100644 moban/data_loader/json.py create mode 100644 moban/data_loader/yaml.py create mode 100644 moban/loaders.py diff --git a/moban/data_loader/__init__.py b/moban/data_loader/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/moban/data_loader/json.py b/moban/data_loader/json.py new file mode 100644 index 00000000..9b6a8500 --- /dev/null +++ b/moban/data_loader/json.py @@ -0,0 +1,15 @@ +import json + +from lml.plugin import PluginInfo + +from moban import constants + + +@PluginInfo(constants.DATA_LOADER_EXTENSION, tags=["yaml", "yml"]) +def open_json(file_name): + """ + returns json contents as string + """ + with open(file_name, "r") as json_data: + data = json.loads(json_data.read()) + return data diff --git a/moban/data_loader/yaml.py b/moban/data_loader/yaml.py new file mode 100644 index 00000000..c73feafb --- /dev/null +++ b/moban/data_loader/yaml.py @@ -0,0 +1,12 @@ +from lml.plugin import PluginInfo +from ruamel.yaml import YAML + +from moban import constants + + +@PluginInfo(constants.DATA_LOADER_EXTENSION, tags=["yaml", "yml"]) +def open_yaml(file_name): + with open(file_name, "r") as data_yaml: + yaml = YAML(typ="rt") + data = yaml.load(data_yaml) + return data diff --git a/moban/loaders.py b/moban/loaders.py new file mode 100644 index 00000000..172c9998 --- /dev/null +++ b/moban/loaders.py @@ -0,0 +1,8 @@ +from lml.plugin import PluginInfo + +from moban import constants + + +class DataLoader(PluginInfo): + def __init__(self): + super(DataLoader, self).__init__(constants.DATA_LOADER_EXTENSION) From a22539a4a0748ef66678f7ee3327529b74e2f2b3 Mon Sep 17 00:00:00 2001 From: chfw Date: Tue, 22 Jan 2019 18:33:47 +0000 Subject: [PATCH 03/38] :sparkles: any data loader. in theory, any data with similar schema could be loaded and used by moban. what's more, 'overrides' syntax is promoted to accept any data format, child.yml could override parent.json. related to #164 --- moban/constants.py | 1 + moban/data_loader/json.py | 2 +- moban/main.py | 7 ++----- moban/plugins.py | 17 +++++++++++------ tests/test_json_loader.py | 11 +++++++++++ tests/test_utils.py | 7 ------- tests/test_yaml_loader.py | 6 ++++-- 7 files changed, 30 insertions(+), 21 deletions(-) create mode 100644 tests/test_json_loader.py diff --git a/moban/constants.py b/moban/constants.py index 7f9cef62..354f6f06 100644 --- a/moban/constants.py +++ b/moban/constants.py @@ -12,6 +12,7 @@ ".%s%s" % (PROGRAM_NAME, ".yaml"), ] DEFAULT_TEMPLATE_TYPE = "jinja2" +DEFAULT_DATA_TYPE = "yaml" # .moban.hashes DEFAULT_MOBAN_CACHE_FILE = ".moban.hashes" diff --git a/moban/data_loader/json.py b/moban/data_loader/json.py index 9b6a8500..b9dd9f94 100644 --- a/moban/data_loader/json.py +++ b/moban/data_loader/json.py @@ -5,7 +5,7 @@ from moban import constants -@PluginInfo(constants.DATA_LOADER_EXTENSION, tags=["yaml", "yml"]) +@PluginInfo(constants.DATA_LOADER_EXTENSION, tags=["json"]) def open_json(file_name): """ returns json contents as string diff --git a/moban/main.py b/moban/main.py index b9f2a65c..c1aa2813 100644 --- a/moban/main.py +++ b/moban/main.py @@ -11,11 +11,8 @@ import sys import argparse -import moban.reporter as reporter -import moban.constants as constants -import moban.mobanfile as mobanfile -import moban.exceptions as exceptions -from moban import plugins +from moban import plugins, reporter, constants, mobanfile, exceptions +from moban.utils import merge from moban.hashstore import HASH_STORE diff --git a/moban/plugins.py b/moban/plugins.py index 0cdb94f0..00d796ea 100644 --- a/moban/plugins.py +++ b/moban/plugins.py @@ -8,7 +8,11 @@ from moban.strategy import Strategy from moban.hashstore import HASH_STORE -BUILTIN_EXENSIONS = ["moban.jinja2.engine"] +BUILTIN_EXENSIONS = [ + "moban.jinja2.engine", + "moban.data_loader.yaml", + "moban.data_loader.json", +] class LibraryManager(PluginManager): @@ -166,17 +170,18 @@ def raise_exception(self, key): class AnyDataLoader(PluginManager): def __init__(self): - super(AnyDataLoader, self).__init__( - constants.DATA_LOADER_EXTENSION - ) + super(AnyDataLoader, self).__init__(constants.DATA_LOADER_EXTENSION) def get_data(self, file_name): file_extension = os.path.splitext(file_name)[1] file_type = file_extension - if file_extension.startswith('.'): + if file_extension.startswith("."): file_type = file_type[1:] - loader_function = self.load_me_now(file_type) + try: + loader_function = self.load_me_now(file_type) + except Exception: + loader_function = self.load_me_now(constants.DEFAULT_DATA_TYPE) return loader_function(file_name) diff --git a/tests/test_json_loader.py b/tests/test_json_loader.py new file mode 100644 index 00000000..c120c7e2 --- /dev/null +++ b/tests/test_json_loader.py @@ -0,0 +1,11 @@ +import os + +from nose.tools import eq_ + +from moban.data_loader.json import open_json + + +def test_open_json(): + content = open_json(os.path.join("tests", "fixtures", "child.json")) + expected = {"key": "hello world", "pass": "ox"} + eq_(expected, content) diff --git a/tests/test_utils.py b/tests/test_utils.py index 609629df..cfa7b12d 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -9,7 +9,6 @@ from moban.utils import ( mkdir_p, - open_json, get_repo_name, write_file_out, file_permissions, @@ -178,9 +177,3 @@ def test_get_repo_name(): actual = [get_repo_name(repo) for repo in repos] expected = ["repo", "repo"] eq_(expected, actual) - - -def test_open_json(): - content = open_json(os.path.join("tests", "fixtures"), "child.json") - expected = {"key": "hello world", "pass": "ox"} - eq_(expected, content) diff --git a/tests/test_yaml_loader.py b/tests/test_yaml_loader.py index 6c74abdf..52e254d5 100644 --- a/tests/test_yaml_loader.py +++ b/tests/test_yaml_loader.py @@ -2,17 +2,19 @@ from nose.tools import eq_, raises -from moban.plugins import load_data +from moban.plugins import load_data, make_sure_all_pkg_are_loaded +from moban.data_loader.yaml import open_yaml def test_simple_yaml(): test_file = os.path.join("tests", "fixtures", "simple.yaml") - data = load_data(os.path.join("tests", "fixtures"), test_file) + data = open_yaml(test_file) eq_(data, {"simple": "yaml"}) def test_inheritance_yaml(): test_file = os.path.join("tests", "fixtures", "child.yaml") + make_sure_all_pkg_are_loaded() data = load_data(os.path.join("tests", "fixtures", "config"), test_file) eq_(data, {"key": "hello world", "pass": "ox"}) From 7c9706fbed83066ad91624feed12d6f7e51e2509 Mon Sep 17 00:00:00 2001 From: chfw Date: Tue, 22 Jan 2019 18:40:37 +0000 Subject: [PATCH 04/38] :hammer: code refactoring on data loader --- .../{data_loader => data_loaders}/__init__.py | 0 moban/{data_loader => data_loaders}/json.py | 0 moban/{data_loader => data_loaders}/yaml.py | 0 moban/loaders.py | 8 ------- moban/plugins.py | 22 +++---------------- tests/test_json_loader.py | 2 +- tests/test_yaml_loader.py | 5 ++--- 7 files changed, 6 insertions(+), 31 deletions(-) rename moban/{data_loader => data_loaders}/__init__.py (100%) rename moban/{data_loader => data_loaders}/json.py (100%) rename moban/{data_loader => data_loaders}/yaml.py (100%) delete mode 100644 moban/loaders.py diff --git a/moban/data_loader/__init__.py b/moban/data_loaders/__init__.py similarity index 100% rename from moban/data_loader/__init__.py rename to moban/data_loaders/__init__.py diff --git a/moban/data_loader/json.py b/moban/data_loaders/json.py similarity index 100% rename from moban/data_loader/json.py rename to moban/data_loaders/json.py diff --git a/moban/data_loader/yaml.py b/moban/data_loaders/yaml.py similarity index 100% rename from moban/data_loader/yaml.py rename to moban/data_loaders/yaml.py diff --git a/moban/loaders.py b/moban/loaders.py deleted file mode 100644 index 172c9998..00000000 --- a/moban/loaders.py +++ /dev/null @@ -1,8 +0,0 @@ -from lml.plugin import PluginInfo - -from moban import constants - - -class DataLoader(PluginInfo): - def __init__(self): - super(DataLoader, self).__init__(constants.DATA_LOADER_EXTENSION) diff --git a/moban/plugins.py b/moban/plugins.py index 00d796ea..85a11a42 100644 --- a/moban/plugins.py +++ b/moban/plugins.py @@ -7,11 +7,12 @@ from moban import utils, constants, exceptions from moban.strategy import Strategy from moban.hashstore import HASH_STORE +from moban.data_loaders.manager import AnyDataLoader BUILTIN_EXENSIONS = [ "moban.jinja2.engine", - "moban.data_loader.yaml", - "moban.data_loader.json", + "moban.data_loaders.yaml", + "moban.data_loaders.json", ] @@ -168,23 +169,6 @@ def raise_exception(self, key): raise exceptions.NoThirdPartyEngine(key) -class AnyDataLoader(PluginManager): - def __init__(self): - super(AnyDataLoader, self).__init__(constants.DATA_LOADER_EXTENSION) - - def get_data(self, file_name): - file_extension = os.path.splitext(file_name)[1] - file_type = file_extension - if file_extension.startswith("."): - file_type = file_type[1:] - - try: - loader_function = self.load_me_now(file_type) - except Exception: - loader_function = self.load_me_now(constants.DEFAULT_DATA_TYPE) - return loader_function(file_name) - - LIBRARIES = LibraryManager() ENGINES = EngineFactory() LOADERS = AnyDataLoader() diff --git a/tests/test_json_loader.py b/tests/test_json_loader.py index c120c7e2..da26c061 100644 --- a/tests/test_json_loader.py +++ b/tests/test_json_loader.py @@ -2,7 +2,7 @@ from nose.tools import eq_ -from moban.data_loader.json import open_json +from moban.data_loaders.json import open_json def test_open_json(): diff --git a/tests/test_yaml_loader.py b/tests/test_yaml_loader.py index 52e254d5..8ff4e160 100644 --- a/tests/test_yaml_loader.py +++ b/tests/test_yaml_loader.py @@ -2,8 +2,8 @@ from nose.tools import eq_, raises -from moban.plugins import load_data, make_sure_all_pkg_are_loaded -from moban.data_loader.yaml import open_yaml +from moban.plugins import load_data +from moban.data_loaders.yaml import open_yaml def test_simple_yaml(): @@ -14,7 +14,6 @@ def test_simple_yaml(): def test_inheritance_yaml(): test_file = os.path.join("tests", "fixtures", "child.yaml") - make_sure_all_pkg_are_loaded() data = load_data(os.path.join("tests", "fixtures", "config"), test_file) eq_(data, {"key": "hello world", "pass": "ox"}) From d87311d349b3a7a25b23bd03804a27fd29e90b52 Mon Sep 17 00:00:00 2001 From: chfw Date: Tue, 22 Jan 2019 20:23:33 +0000 Subject: [PATCH 05/38] :newspaper: add missing file --- moban/data_loaders/manager.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 moban/data_loaders/manager.py diff --git a/moban/data_loaders/manager.py b/moban/data_loaders/manager.py new file mode 100644 index 00000000..eb7970e9 --- /dev/null +++ b/moban/data_loaders/manager.py @@ -0,0 +1,20 @@ +import os +from lml.plugin import PluginManager +from moban import constants + + +class AnyDataLoader(PluginManager): + def __init__(self): + super(AnyDataLoader, self).__init__(constants.DATA_LOADER_EXTENSION) + + def get_data(self, file_name): + file_extension = os.path.splitext(file_name)[1] + file_type = file_extension + if file_extension.startswith("."): + file_type = file_type[1:] + + try: + loader_function = self.load_me_now(file_type) + except Exception: + loader_function = self.load_me_now(constants.DEFAULT_DATA_TYPE) + return loader_function(file_name) From 2e6aefc69fda84a8f993cb17e486f5293f18bcf5 Mon Sep 17 00:00:00 2001 From: chfw Date: Tue, 22 Jan 2019 20:33:54 +0000 Subject: [PATCH 06/38] :microscope: fix unit test failure on python 2 --- moban/data_loaders/json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moban/data_loaders/json.py b/moban/data_loaders/json.py index b9dd9f94..30f57268 100644 --- a/moban/data_loaders/json.py +++ b/moban/data_loaders/json.py @@ -11,5 +11,5 @@ def open_json(file_name): returns json contents as string """ with open(file_name, "r") as json_data: - data = json.loads(json_data.read()) + data = json.load(json_data) return data From a309bf7484bb8fea458eb09ee2b615c2011b05f8 Mon Sep 17 00:00:00 2001 From: CLiu13 Date: Sat, 19 Jan 2019 16:29:37 -0500 Subject: [PATCH 07/38] =?UTF-8?q?=E2=9C=A8=20Add=20`-v`=20to=20show=20curr?= =?UTF-8?q?ent=20moban=20version?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes https://github.com/moremoban/moban/issues/107 --- .moban.cd/changelog.yml | 6 ++++++ CHANGELOG.rst | 9 +++++++++ moban/constants.py | 1 + moban/main.py | 7 +++++++ tests/integration_tests/test_command_line_options.py | 9 +++++++++ 5 files changed, 32 insertions(+) diff --git a/.moban.cd/changelog.yml b/.moban.cd/changelog.yml index 2f28e686..519d782c 100644 --- a/.moban.cd/changelog.yml +++ b/.moban.cd/changelog.yml @@ -1,6 +1,12 @@ name: moban organisation: moremoban releases: +- changes: + - action: Updated + details: + - "`#107`: Add -v to show current moban version" + date: 22-1-2019 + version: 0.4.0 - changes: - action: Updated details: diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ac728d82..1239b428 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,15 @@ Change log ================================================================================ +0.4.0 - 22-1-2019 +-------------------------------------------------------------------------------- + +Updated +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +#. `#107 `_: Add -v to show + current moban version + 0.3.9 - 18-1-2019 -------------------------------------------------------------------------------- diff --git a/moban/constants.py b/moban/constants.py index b17eeaca..bed9ab9e 100644 --- a/moban/constants.py +++ b/moban/constants.py @@ -29,6 +29,7 @@ LABEL_OVERRIDES = "overrides" LABEL_MOBANFILE = "mobanfile" LABEL_FORCE = "force" +LABEL_VERSION = "version" DEFAULT_CONFIGURATION_DIRNAME = ".moban.cd" diff --git a/moban/main.py b/moban/main.py index f811efa0..5429edd2 100644 --- a/moban/main.py +++ b/moban/main.py @@ -18,6 +18,7 @@ from moban import plugins from moban.utils import merge, open_yaml from moban.hashstore import HASH_STORE +from moban._version import __version__ def main(): @@ -115,6 +116,12 @@ def create_parser(): nargs="?", help="string templates", ) + parser.add_argument( + "-v", + "--%s" % constants.LABEL_VERSION, + action="version", + version="%(prog)s {v}".format(v=__version__) + ) return parser diff --git a/tests/integration_tests/test_command_line_options.py b/tests/integration_tests/test_command_line_options.py index 666fe3f8..d8bf7fd1 100644 --- a/tests/integration_tests/test_command_line_options.py +++ b/tests/integration_tests/test_command_line_options.py @@ -408,3 +408,12 @@ def test_mako_option(self, fake_template_doer): def tearDown(self): os.unlink(self.config_file) + + +@raises(SystemExit) +def test_version_option(): + test_args = ["moban", "-v"] + with patch.object(sys, "argv", test_args): + from moban.main import main + + main() From e1b2782cdc556a1164e0f235558e517f1e0a8ca4 Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 23 Jan 2019 09:02:48 +0000 Subject: [PATCH 08/38] :books: update documentation --- docs/README.rst | 5 ++ .../.moban.cd/parent.json | 4 ++ .../.moban.cd/parent.yaml | 2 + .../.moban.td/base.jj2 | 9 ++++ .../README.rst | 51 +++++++++++++++++++ .../a.template | 9 ++++ .../child.json | 4 ++ .../child.yaml | 2 + docs/level-3-data-override/README.rst | 2 +- tests/test_docs.py | 26 ++++++++++ 10 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 docs/level-13-any-data-override-any-data/.moban.cd/parent.json create mode 100644 docs/level-13-any-data-override-any-data/.moban.cd/parent.yaml create mode 100644 docs/level-13-any-data-override-any-data/.moban.td/base.jj2 create mode 100644 docs/level-13-any-data-override-any-data/README.rst create mode 100644 docs/level-13-any-data-override-any-data/a.template create mode 100644 docs/level-13-any-data-override-any-data/child.json create mode 100644 docs/level-13-any-data-override-any-data/child.yaml diff --git a/docs/README.rst b/docs/README.rst index d3cafab8..9ae18452 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -14,6 +14,8 @@ This section covers the use cases for moban. Please check them out individually. #. `Use pypi package as a moban dependency`_ #. `Use git repository as a moban dependency`_ #. `Use handlebars template with moban`_ +#. `Use template engine extensions`_ +#. `Any data overrides any data`_ .. _Jinja2 command line: level-1-jinja2-cli .. _Template inheritance: level-2-template-inheritance @@ -26,3 +28,6 @@ This section covers the use cases for moban. Please check them out individually. .. _Use pypi package as a moban dependency: level-9-moban-dependency-as-pypi-package .. _Use git repository as a moban dependency: level-10-moban-dependency-as-git-repo .. _Use handlebars template with moban: level-11-use-handlebars +.. _Use template engine extensions: level-12-use-template-engine-extensions +.. _Any data overrides any data: level-13-any-data-override-any-data + diff --git a/docs/level-13-any-data-override-any-data/.moban.cd/parent.json b/docs/level-13-any-data-override-any-data/.moban.cd/parent.json new file mode 100644 index 00000000..275cdf70 --- /dev/null +++ b/docs/level-13-any-data-override-any-data/.moban.cd/parent.json @@ -0,0 +1,4 @@ +{ + "nihao": "shijie from parent.json", + "hello": "shijie from parent.json" +} diff --git a/docs/level-13-any-data-override-any-data/.moban.cd/parent.yaml b/docs/level-13-any-data-override-any-data/.moban.cd/parent.yaml new file mode 100644 index 00000000..ff67cd90 --- /dev/null +++ b/docs/level-13-any-data-override-any-data/.moban.cd/parent.yaml @@ -0,0 +1,2 @@ +nihao: shijie from parent.yaml +hello: shijie \ No newline at end of file diff --git a/docs/level-13-any-data-override-any-data/.moban.td/base.jj2 b/docs/level-13-any-data-override-any-data/.moban.td/base.jj2 new file mode 100644 index 00000000..1f84d223 --- /dev/null +++ b/docs/level-13-any-data-override-any-data/.moban.td/base.jj2 @@ -0,0 +1,9 @@ +{%block header %} +{%endblock%} + +{{hello}} + +{{nihao}} + +{%block footer %} +{%endblock%} \ No newline at end of file diff --git a/docs/level-13-any-data-override-any-data/README.rst b/docs/level-13-any-data-override-any-data/README.rst new file mode 100644 index 00000000..ae1bc6f5 --- /dev/null +++ b/docs/level-13-any-data-override-any-data/README.rst @@ -0,0 +1,51 @@ +Level 13: any data override any data +================================================================================ + +It's thought that why shall we constrain ourselves on yaml file format. Along +the development path, json file format was added. What about other file formats? + +Hence `moban` v0.4.0 allow data loader extension, and by default yaml, json +is supported. Due to the new capability `overrides` key word can override any +data format:: + + overrides: data.base.json + .... + + +Evaluation +-------------------------------------------------------------------------------- + +Please change directory to `docs/level-13-any-data-override-any-data` directory. + +In this example, `child.yaml` overrides `.moban.cd/parent.json`, here is the +command to launch it: + +.. code-block:: bash + + moban -c child.yaml -t a.template + +'moban.output' is the generated file:: + + ========header============ + + world from child.yaml + + shijie from parent.json + + ========footer============ + + +And we can try `child.json`, which you can guess, overrides `.moban.cd/parent.yaml`: +.. code-block:: bash + + moban -c child.json -t a.template + +'moban.output' is the generated file:: + + ========header============ + + world from child.json + + shijie from parent.yml + + ========footer============ diff --git a/docs/level-13-any-data-override-any-data/a.template b/docs/level-13-any-data-override-any-data/a.template new file mode 100644 index 00000000..44672595 --- /dev/null +++ b/docs/level-13-any-data-override-any-data/a.template @@ -0,0 +1,9 @@ +{%extends 'base.jj2' %} + +{%block header %} +========header============ +{%endblock%} + +{%block footer %} +========footer============ +{%endblock%} diff --git a/docs/level-13-any-data-override-any-data/child.json b/docs/level-13-any-data-override-any-data/child.json new file mode 100644 index 00000000..52d09210 --- /dev/null +++ b/docs/level-13-any-data-override-any-data/child.json @@ -0,0 +1,4 @@ +{ + "overrides": "parent.yaml", + "hello": "world from child.json" +} diff --git a/docs/level-13-any-data-override-any-data/child.yaml b/docs/level-13-any-data-override-any-data/child.yaml new file mode 100644 index 00000000..1a36dfbb --- /dev/null +++ b/docs/level-13-any-data-override-any-data/child.yaml @@ -0,0 +1,2 @@ +overrides: parent.json +hello: world from child.yaml diff --git a/docs/level-3-data-override/README.rst b/docs/level-3-data-override/README.rst index 071b32b4..c163c38a 100644 --- a/docs/level-3-data-override/README.rst +++ b/docs/level-3-data-override/README.rst @@ -21,7 +21,7 @@ command to launch it: moban -c data.yaml -t a.template -'a.output' is the generated file:: +'moban.output' is the generated file:: ========header============ diff --git a/tests/test_docs.py b/tests/test_docs.py index 99c5bad1..0cdf24e6 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -124,6 +124,32 @@ def test_level_12_b(self): folder = "level-12-use-template-engine-extensions" self._raw_moban(["moban"], folder, expected_b, "b.output") + def test_level_13_json(self): + expected = """========header============ + +world from child.json + +shijie from parent.yaml + +========footer============ +""" + folder = "level-13-any-data-override-any-data" + commands = ["moban", "-c", "child.json", "-t", "a.template"] + self._raw_moban(commands, folder, expected, "moban.output") + + def test_level_13_yaml(self): + expected = """========header============ + +world from child.yaml + +shijie from parent.json + +========footer============ +""" + folder = "level-13-any-data-override-any-data" + commands = ["moban", "-c", "child.yaml", "-t", "a.template"] + self._raw_moban(commands, folder, expected, "moban.output") + def test_misc_1(self): expected = "test file\n" From 1aaba7fbb78fa2fa5ffc8ce190bc6b0cdd00234b Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 24 Jan 2019 07:41:03 +0000 Subject: [PATCH 09/38] :books: update tutorial on data loader extensions --- docs/README.rst | 3 +- .../README.rst | 15 +++-- .../.moban.cd/parent.custom | 2 + .../.moban.cd/parent.json | 4 ++ .../.moban.td/base.jj2 | 9 +++ docs/level-14-custom-data-loader/.moban.yml | 9 +++ docs/level-14-custom-data-loader/README.rst | 65 +++++++++++++++++++ docs/level-14-custom-data-loader/a.template | 9 +++ docs/level-14-custom-data-loader/b.output | 7 ++ docs/level-14-custom-data-loader/child.custom | 2 + .../custom-data-loaders/custom.py | 16 +++++ .../override_custom.yaml | 2 + tests/test_docs.py | 13 ++++ 13 files changed, 148 insertions(+), 8 deletions(-) create mode 100644 docs/level-14-custom-data-loader/.moban.cd/parent.custom create mode 100644 docs/level-14-custom-data-loader/.moban.cd/parent.json create mode 100644 docs/level-14-custom-data-loader/.moban.td/base.jj2 create mode 100644 docs/level-14-custom-data-loader/.moban.yml create mode 100644 docs/level-14-custom-data-loader/README.rst create mode 100644 docs/level-14-custom-data-loader/a.template create mode 100644 docs/level-14-custom-data-loader/b.output create mode 100644 docs/level-14-custom-data-loader/child.custom create mode 100644 docs/level-14-custom-data-loader/custom-data-loaders/custom.py create mode 100644 docs/level-14-custom-data-loader/override_custom.yaml diff --git a/docs/README.rst b/docs/README.rst index 9ae18452..960a3562 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -16,6 +16,7 @@ This section covers the use cases for moban. Please check them out individually. #. `Use handlebars template with moban`_ #. `Use template engine extensions`_ #. `Any data overrides any data`_ +#. `Custom data loader`_ .. _Jinja2 command line: level-1-jinja2-cli .. _Template inheritance: level-2-template-inheritance @@ -30,4 +31,4 @@ This section covers the use cases for moban. Please check them out individually. .. _Use handlebars template with moban: level-11-use-handlebars .. _Use template engine extensions: level-12-use-template-engine-extensions .. _Any data overrides any data: level-13-any-data-override-any-data - +.. _Custom data loader: level-14-custom-data-loader diff --git a/docs/level-13-any-data-override-any-data/README.rst b/docs/level-13-any-data-override-any-data/README.rst index ae1bc6f5..a6e5d102 100644 --- a/docs/level-13-any-data-override-any-data/README.rst +++ b/docs/level-13-any-data-override-any-data/README.rst @@ -4,13 +4,13 @@ Level 13: any data override any data It's thought that why shall we constrain ourselves on yaml file format. Along the development path, json file format was added. What about other file formats? -Hence `moban` v0.4.0 allow data loader extension, and by default yaml, json -is supported. Due to the new capability `overrides` key word can override any -data format:: +By default yaml, json is supported. Due to the new capability `overrides` key +word can override any supported data format:: overrides: data.base.json .... +or simple use `.json` data instead of `.yaml` data. Evaluation -------------------------------------------------------------------------------- @@ -27,15 +27,16 @@ command to launch it: 'moban.output' is the generated file:: ========header============ - + world from child.yaml - + shijie from parent.json - + ========footer============ -And we can try `child.json`, which you can guess, overrides `.moban.cd/parent.yaml`: +And we can try `child.json`, which you can guess, overrides `.moban.cd/parent.yaml` + .. code-block:: bash moban -c child.json -t a.template diff --git a/docs/level-14-custom-data-loader/.moban.cd/parent.custom b/docs/level-14-custom-data-loader/.moban.cd/parent.custom new file mode 100644 index 00000000..70d3e256 --- /dev/null +++ b/docs/level-14-custom-data-loader/.moban.cd/parent.custom @@ -0,0 +1,2 @@ +hello,nihao +world from parent.cusom,shijie from parent.custom \ No newline at end of file diff --git a/docs/level-14-custom-data-loader/.moban.cd/parent.json b/docs/level-14-custom-data-loader/.moban.cd/parent.json new file mode 100644 index 00000000..275cdf70 --- /dev/null +++ b/docs/level-14-custom-data-loader/.moban.cd/parent.json @@ -0,0 +1,4 @@ +{ + "nihao": "shijie from parent.json", + "hello": "shijie from parent.json" +} diff --git a/docs/level-14-custom-data-loader/.moban.td/base.jj2 b/docs/level-14-custom-data-loader/.moban.td/base.jj2 new file mode 100644 index 00000000..1f84d223 --- /dev/null +++ b/docs/level-14-custom-data-loader/.moban.td/base.jj2 @@ -0,0 +1,9 @@ +{%block header %} +{%endblock%} + +{{hello}} + +{{nihao}} + +{%block footer %} +{%endblock%} \ No newline at end of file diff --git a/docs/level-14-custom-data-loader/.moban.yml b/docs/level-14-custom-data-loader/.moban.yml new file mode 100644 index 00000000..971ce7b2 --- /dev/null +++ b/docs/level-14-custom-data-loader/.moban.yml @@ -0,0 +1,9 @@ +configuration: + plugin_dir: + - custom-data-loaders + template: a.template +targets: + - output: a.output + configuration: child.custom + - output: b.output + configuration: override_custom.yaml \ No newline at end of file diff --git a/docs/level-14-custom-data-loader/README.rst b/docs/level-14-custom-data-loader/README.rst new file mode 100644 index 00000000..e848b47c --- /dev/null +++ b/docs/level-14-custom-data-loader/README.rst @@ -0,0 +1,65 @@ +Level 14: custom data loader +================================================================================ + +Continuing from level 13, `moban` since v0.4.0 allows data loader extension. +Due to the new capability `overrides` key word can override any +data format:: + + overrides: yours.custom + .... + +or simple use `.custom` data instead of `.yaml` data. + +However, you will need to provide a data loader for `.custom` yourselves. + +Evaluation +-------------------------------------------------------------------------------- + +Please change directory to `docs/level-14-custom-data-loader` directory. + + +In this tutorial, a custom data loader was provided to show case its dataloader +extension. Here is the mobanfile:: + + configuration: + plugin_dir: + - custom-data-loaders + template: a.template + targets: + - output: a.output + configuration: child.custom + - output: b.output + configuration: override_custom.yaml + +`custom-data-loaders` is a directory where custom.py lives. The protocol is +that the custom loader register itself to a file extension and return +a data dictionary confirming mobanfile schema. On call, `moban` will provide +an absolute file name for your loader to work on. + + +Here is the code to do the registration: + +.. code-block:: python + + @PluginInfo(constants.DATA_LOADER_EXTENSION, tags=["custom"]) + + +In order to evaluate, you can simply type:: + + $ moban + $ cat a.output + ========header============ + + world from child.cusom + + shijie from parent.json + + ========footer============ + $ cat b.output + ========header============ + + world from override_custom.yaml + + shijie from parent.custom + + ========footer============ diff --git a/docs/level-14-custom-data-loader/a.template b/docs/level-14-custom-data-loader/a.template new file mode 100644 index 00000000..44672595 --- /dev/null +++ b/docs/level-14-custom-data-loader/a.template @@ -0,0 +1,9 @@ +{%extends 'base.jj2' %} + +{%block header %} +========header============ +{%endblock%} + +{%block footer %} +========footer============ +{%endblock%} diff --git a/docs/level-14-custom-data-loader/b.output b/docs/level-14-custom-data-loader/b.output new file mode 100644 index 00000000..48ab2cac --- /dev/null +++ b/docs/level-14-custom-data-loader/b.output @@ -0,0 +1,7 @@ +========header============ + +world from override_custom.yaml + +shijie from parent.custom + +========footer============ diff --git a/docs/level-14-custom-data-loader/child.custom b/docs/level-14-custom-data-loader/child.custom new file mode 100644 index 00000000..25ceefe9 --- /dev/null +++ b/docs/level-14-custom-data-loader/child.custom @@ -0,0 +1,2 @@ +hello,overrides +world from child.cusom,parent.json \ No newline at end of file diff --git a/docs/level-14-custom-data-loader/custom-data-loaders/custom.py b/docs/level-14-custom-data-loader/custom-data-loaders/custom.py new file mode 100644 index 00000000..7a730f4d --- /dev/null +++ b/docs/level-14-custom-data-loader/custom-data-loaders/custom.py @@ -0,0 +1,16 @@ +from lml.plugin import PluginInfo +import csv + +from moban import constants + + +@PluginInfo(constants.DATA_LOADER_EXTENSION, tags=["custom"]) +def open_custom(file_name): + with open(file_name, "r") as data_csv: + csvreader = csv.reader(data_csv) + rows = [] + for row in csvreader: + rows.append(row) + + data = dict(zip(rows[0], rows[1])) + return data diff --git a/docs/level-14-custom-data-loader/override_custom.yaml b/docs/level-14-custom-data-loader/override_custom.yaml new file mode 100644 index 00000000..d24ae6b5 --- /dev/null +++ b/docs/level-14-custom-data-loader/override_custom.yaml @@ -0,0 +1,2 @@ +overrides: parent.custom +hello: world from override_custom.yaml \ No newline at end of file diff --git a/tests/test_docs.py b/tests/test_docs.py index 0cdf24e6..7a5831dd 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -150,6 +150,19 @@ def test_level_13_yaml(self): commands = ["moban", "-c", "child.yaml", "-t", "a.template"] self._raw_moban(commands, folder, expected, "moban.output") + def test_level_14_custom(self): + expected = """========header============ + +world from child.cusom + +shijie from parent.json + +========footer============ +""" + folder = "level-14-custom-data-loader" + commands = ["moban"] + self._raw_moban(commands, folder, expected, "a.output") + def test_misc_1(self): expected = "test file\n" From b064658808ee49ba70632848b1bf468c6a9172a0 Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 24 Jan 2019 08:12:24 +0000 Subject: [PATCH 10/38] :bug: python 2.7 thinks json.py is built-in json module --- moban/data_loaders/{json.py => json_loader.py} | 0 moban/plugins.py | 2 +- tests/test_json_loader.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename moban/data_loaders/{json.py => json_loader.py} (100%) diff --git a/moban/data_loaders/json.py b/moban/data_loaders/json_loader.py similarity index 100% rename from moban/data_loaders/json.py rename to moban/data_loaders/json_loader.py diff --git a/moban/plugins.py b/moban/plugins.py index 85a11a42..68591f59 100644 --- a/moban/plugins.py +++ b/moban/plugins.py @@ -12,7 +12,7 @@ BUILTIN_EXENSIONS = [ "moban.jinja2.engine", "moban.data_loaders.yaml", - "moban.data_loaders.json", + "moban.data_loaders.json_loader", ] diff --git a/tests/test_json_loader.py b/tests/test_json_loader.py index da26c061..fc38acd2 100644 --- a/tests/test_json_loader.py +++ b/tests/test_json_loader.py @@ -2,7 +2,7 @@ from nose.tools import eq_ -from moban.data_loaders.json import open_json +from moban.data_loaders.json_loader import open_json def test_open_json(): From 21183fafdb521e1cd83018618e38f9056b3d716e Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 24 Jan 2019 08:16:31 +0000 Subject: [PATCH 11/38] :books: update change log and pump version number --- .moban.cd/changelog.yml | 1 + .moban.cd/moban.yml | 4 ++-- CHANGELOG.rst | 2 ++ docs/conf.py | 2 +- moban/_version.py | 2 +- setup.py | 2 +- 6 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.moban.cd/changelog.yml b/.moban.cd/changelog.yml index 519d782c..9bc95333 100644 --- a/.moban.cd/changelog.yml +++ b/.moban.cd/changelog.yml @@ -5,6 +5,7 @@ releases: - action: Updated details: - "`#107`: Add -v to show current moban version" + - "`#164`: support additional data formats" date: 22-1-2019 version: 0.4.0 - changes: diff --git a/.moban.cd/moban.yml b/.moban.cd/moban.yml index e049536f..bd2b9205 100644 --- a/.moban.cd/moban.yml +++ b/.moban.cd/moban.yml @@ -3,8 +3,8 @@ organisation: moremoban author: C. W. contact: wangc_2011@hotmail.com license: MIT -version: 0.3.9 -current_version: 0.3.9 +version: 0.4.0 +current_version: 0.4.0 release: 0.3.9 branch: master command_line_interface: "moban" diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1239b428..d8d21ee2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,6 +9,8 @@ Updated #. `#107 `_: Add -v to show current moban version +#. `#164 `_: support additional + data formats 0.3.9 - 18-1-2019 -------------------------------------------------------------------------------- diff --git a/docs/conf.py b/docs/conf.py index 2049721a..a5d48bef 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -30,7 +30,7 @@ # The short X.Y version version = u'0.3.9' # The full version, including alpha/beta/rc tags -release = u'0.3.9' +release = u'0.4.0' # -- General configuration --------------------------------------------------- diff --git a/moban/_version.py b/moban/_version.py index bef32dc0..44e5b7dc 100644 --- a/moban/_version.py +++ b/moban/_version.py @@ -1,2 +1,2 @@ -__version__ = "0.3.9" +__version__ = "0.4.0" __author__ = "C. W." diff --git a/setup.py b/setup.py index 4967639a..01d91030 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ NAME = 'moban' AUTHOR = 'C. W.' -VERSION = '0.3.9' +VERSION = '0.4.0' EMAIL = 'wangc_2011@hotmail.com' LICENSE = 'MIT' ENTRY_POINTS = { From 5c1d5bfa6990e7592fc9c3c82c85ca7dfc32c30d Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 24 Jan 2019 08:20:56 +0000 Subject: [PATCH 12/38] :bug: Python 2 hates module name with '-' mixed --- docs/level-14-custom-data-loader/.moban.yml | 4 ++-- docs/level-14-custom-data-loader/README.rst | 9 ++++++++- .../custom.py | 0 3 files changed, 10 insertions(+), 3 deletions(-) rename docs/level-14-custom-data-loader/{custom-data-loaders => custom_data_loaders}/custom.py (100%) diff --git a/docs/level-14-custom-data-loader/.moban.yml b/docs/level-14-custom-data-loader/.moban.yml index 971ce7b2..db37f920 100644 --- a/docs/level-14-custom-data-loader/.moban.yml +++ b/docs/level-14-custom-data-loader/.moban.yml @@ -1,9 +1,9 @@ configuration: plugin_dir: - - custom-data-loaders + - custom_data_loaders template: a.template targets: - output: a.output configuration: child.custom - output: b.output - configuration: override_custom.yaml \ No newline at end of file + configuration: override_custom.yaml diff --git a/docs/level-14-custom-data-loader/README.rst b/docs/level-14-custom-data-loader/README.rst index e848b47c..96842c3c 100644 --- a/docs/level-14-custom-data-loader/README.rst +++ b/docs/level-14-custom-data-loader/README.rst @@ -31,7 +31,7 @@ extension. Here is the mobanfile:: - output: b.output configuration: override_custom.yaml -`custom-data-loaders` is a directory where custom.py lives. The protocol is +`custom_data_loaders` is a directory where custom.py lives. The protocol is that the custom loader register itself to a file extension and return a data dictionary confirming mobanfile schema. On call, `moban` will provide an absolute file name for your loader to work on. @@ -63,3 +63,10 @@ In order to evaluate, you can simply type:: shijie from parent.custom ========footer============ + + +.. note:: + + Python 2 does not like plugin directory name to have dash, '-' in module names. + In other words, Python 2 does not like 'custom-data-loaders' but accept + 'custom_data_loaders'. diff --git a/docs/level-14-custom-data-loader/custom-data-loaders/custom.py b/docs/level-14-custom-data-loader/custom_data_loaders/custom.py similarity index 100% rename from docs/level-14-custom-data-loader/custom-data-loaders/custom.py rename to docs/level-14-custom-data-loader/custom_data_loaders/custom.py From e1d07fd4d2d2d067d6533bc703ec7399bee0e8ea Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 24 Jan 2019 08:25:36 +0000 Subject: [PATCH 13/38] :fire: remove python 3.4 as test target --- .moban.d/travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.moban.d/travis.yml b/.moban.d/travis.yml index a7ca546e..3408a1ba 100644 --- a/.moban.d/travis.yml +++ b/.moban.d/travis.yml @@ -6,6 +6,5 @@ python: - 3.7-dev - 3.6 - 3.5 - - 3.4 - 2.7 {%endblock%} From d28718b68e55424e54781ea9a864004e34872851 Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 24 Jan 2019 18:00:10 +0000 Subject: [PATCH 14/38] :bug: python 2.7 wanted __init__.py in plugin dir --- docs/level-14-custom-data-loader/custom_data_loaders/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 docs/level-14-custom-data-loader/custom_data_loaders/__init__.py diff --git a/docs/level-14-custom-data-loader/custom_data_loaders/__init__.py b/docs/level-14-custom-data-loader/custom_data_loaders/__init__.py new file mode 100644 index 00000000..e69de29b From 7e721e8e1e8447a34ae0f86ed874e4f5e3431597 Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 24 Jan 2019 18:05:51 +0000 Subject: [PATCH 15/38] :fire: remove the unwanted folder name change and :books: update the demand of python 2 on custom plugin dirs --- docs/level-14-custom-data-loader/.moban.yml | 2 +- docs/level-14-custom-data-loader/README.rst | 11 +++++------ .../__init__.py | 0 .../custom.py | 0 4 files changed, 6 insertions(+), 7 deletions(-) rename docs/level-14-custom-data-loader/{custom_data_loaders => custom-data-loader}/__init__.py (100%) rename docs/level-14-custom-data-loader/{custom_data_loaders => custom-data-loader}/custom.py (100%) diff --git a/docs/level-14-custom-data-loader/.moban.yml b/docs/level-14-custom-data-loader/.moban.yml index db37f920..1da8b4ba 100644 --- a/docs/level-14-custom-data-loader/.moban.yml +++ b/docs/level-14-custom-data-loader/.moban.yml @@ -1,6 +1,6 @@ configuration: plugin_dir: - - custom_data_loaders + - custom-data-loaders template: a.template targets: - output: a.output diff --git a/docs/level-14-custom-data-loader/README.rst b/docs/level-14-custom-data-loader/README.rst index 96842c3c..7568c33a 100644 --- a/docs/level-14-custom-data-loader/README.rst +++ b/docs/level-14-custom-data-loader/README.rst @@ -23,7 +23,7 @@ extension. Here is the mobanfile:: configuration: plugin_dir: - - custom-data-loaders + - custom-data-loader template: a.template targets: - output: a.output @@ -31,7 +31,7 @@ extension. Here is the mobanfile:: - output: b.output configuration: override_custom.yaml -`custom_data_loaders` is a directory where custom.py lives. The protocol is +`custom-data-loader` is a directory where custom.py lives. The protocol is that the custom loader register itself to a file extension and return a data dictionary confirming mobanfile schema. On call, `moban` will provide an absolute file name for your loader to work on. @@ -65,8 +65,7 @@ In order to evaluate, you can simply type:: ========footer============ -.. note:: +.. warning:: - Python 2 does not like plugin directory name to have dash, '-' in module names. - In other words, Python 2 does not like 'custom-data-loaders' but accept - 'custom_data_loaders'. + Python 2 dictates the existence of __init__.py in the plugin directory. Otheriwse + your plugin won't load diff --git a/docs/level-14-custom-data-loader/custom_data_loaders/__init__.py b/docs/level-14-custom-data-loader/custom-data-loader/__init__.py similarity index 100% rename from docs/level-14-custom-data-loader/custom_data_loaders/__init__.py rename to docs/level-14-custom-data-loader/custom-data-loader/__init__.py diff --git a/docs/level-14-custom-data-loader/custom_data_loaders/custom.py b/docs/level-14-custom-data-loader/custom-data-loader/custom.py similarity index 100% rename from docs/level-14-custom-data-loader/custom_data_loaders/custom.py rename to docs/level-14-custom-data-loader/custom-data-loader/custom.py From 55e8d50b91fd7f70d517a3fcc791780f597b5a09 Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 24 Jan 2019 18:19:54 +0000 Subject: [PATCH 16/38] :hammer: refactor test docs code --- tests/test_docs.py | 189 +++++++++++++++++++++++++++------------------ 1 file changed, 115 insertions(+), 74 deletions(-) diff --git a/tests/test_docs.py b/tests/test_docs.py index 7a5831dd..be034ced 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -1,5 +1,6 @@ import os import sys +from textwrap import dedent from mock import patch from nose.tools import eq_ @@ -7,6 +8,13 @@ from moban.main import main +def custom_dedent(long_texts): + refined = dedent(long_texts) + if refined.startswith("\n"): + refined = refined[1:] + return refined + + class TestTutorial: def setUp(self): self.current = os.getcwd() @@ -17,72 +25,90 @@ def test_level_1(self): self._moban(folder, expected) def test_level_2(self): - expected = """========header============ + expected = custom_dedent( + """ + ========header============ -world + world -========footer============ -""" + ========footer============ + """ + ) folder = "level-2-template-inheritance" self._moban(folder, expected) def test_level_3(self): - expected = """========header============ + expected = custom_dedent( + """ + ========header============ -world + world -shijie + shijie -========footer============ -""" + ========footer============ + """ + ) folder = "level-3-data-override" self._moban(folder, expected) def test_level_4(self): - expected = """========header============ - -world - -shijie - -========footer============ -""" + expected = custom_dedent( + """ + ========header============ + + world + + shijie + + ========footer============ + """ + ) folder = "level-4-single-command" self._raw_moban(["moban"], folder, expected, "a.output") def test_level_5(self): - expected = """========header============ - -world - -shijie - -this demonstrates jinja2's include statement - -========footer============ -""" + expected = custom_dedent( + """ + ========header============ + + world + + shijie + + this demonstrates jinja2's include statement + + ========footer============ + """ + ) folder = "level-5-custom-configuration" self._raw_moban(["moban"], folder, expected, "a.output") def test_level_6(self): - expected = """========header============ - -world2 - -shijie - -this demonstrates jinja2's include statement - -========footer============ -""" + expected = custom_dedent( + """ + ========header============ + + world2 + + shijie + + this demonstrates jinja2's include statement + + ========footer============ + """ + ) folder = "level-6-complex-configuration" self._raw_moban(["moban"], folder, expected, "a.output2") def test_level_7(self): - expected = """Hello, you are in level 7 example - -Hello, you are not in level 7 -""" + expected = custom_dedent( + """ + Hello, you are in level 7 example + + Hello, you are not in level 7 + """ + ) folder = "level-7-use-custom-jinja2-filter-test-n-global" self._raw_moban(["moban"], folder, expected, "test.output") @@ -108,57 +134,72 @@ def test_level_11(self): self._raw_moban(["moban"], folder, expected, "a.output") def test_level_12_a(self): - expected_a = """world -world -world -world -""" + expected_a = custom_dedent( + """ + world + world + world + world + """ + ) folder = "level-12-use-template-engine-extensions" self._raw_moban(["moban"], folder, expected_a, "a.output") def test_level_12_b(self): - expected_b = """142 -42 -142 -""" + expected_b = custom_dedent( + """ + 142 + 42 + 142 + """ + ) folder = "level-12-use-template-engine-extensions" self._raw_moban(["moban"], folder, expected_b, "b.output") def test_level_13_json(self): - expected = """========header============ - -world from child.json - -shijie from parent.yaml - -========footer============ -""" + expected = custom_dedent( + """ + ========header============ + + world from child.json + + shijie from parent.yaml + + ========footer============ + """ + ) folder = "level-13-any-data-override-any-data" commands = ["moban", "-c", "child.json", "-t", "a.template"] self._raw_moban(commands, folder, expected, "moban.output") def test_level_13_yaml(self): - expected = """========header============ - -world from child.yaml - -shijie from parent.json - -========footer============ -""" + expected = custom_dedent( + """ + ========header============ + + world from child.yaml + + shijie from parent.json + + ========footer============ + """ + ) folder = "level-13-any-data-override-any-data" commands = ["moban", "-c", "child.yaml", "-t", "a.template"] self._raw_moban(commands, folder, expected, "moban.output") def test_level_14_custom(self): - expected = """========header============ - -world from child.cusom - -shijie from parent.json - -========footer============ -""" + expected = custom_dedent( + """ + ========header============ + + world from child.cusom + + shijie from parent.json + + ========footer============ + """ + ) folder = "level-14-custom-data-loader" commands = ["moban"] self._raw_moban(commands, folder, expected, "a.output") From 77a6eb5e28ab4ffc860525b66f1ddefe939ad20b Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 24 Jan 2019 18:20:40 +0000 Subject: [PATCH 17/38] :bug: update custom data loader plugin directory name --- docs/level-14-custom-data-loader/.moban.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/level-14-custom-data-loader/.moban.yml b/docs/level-14-custom-data-loader/.moban.yml index 1da8b4ba..6058d6c2 100644 --- a/docs/level-14-custom-data-loader/.moban.yml +++ b/docs/level-14-custom-data-loader/.moban.yml @@ -1,6 +1,6 @@ configuration: plugin_dir: - - custom-data-loaders + - custom-data-loader template: a.template targets: - output: a.output From 45681241b9d96d348764d16827054bfec2771952 Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 24 Jan 2019 18:20:55 +0000 Subject: [PATCH 18/38] :hammer: reformat code --- docs/level-14-custom-data-loader/custom-data-loader/custom.py | 3 ++- moban/data_loaders/manager.py | 2 ++ moban/main.py | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/level-14-custom-data-loader/custom-data-loader/custom.py b/docs/level-14-custom-data-loader/custom-data-loader/custom.py index 7a730f4d..61813cce 100644 --- a/docs/level-14-custom-data-loader/custom-data-loader/custom.py +++ b/docs/level-14-custom-data-loader/custom-data-loader/custom.py @@ -1,6 +1,7 @@ -from lml.plugin import PluginInfo import csv +from lml.plugin import PluginInfo + from moban import constants diff --git a/moban/data_loaders/manager.py b/moban/data_loaders/manager.py index eb7970e9..280499d9 100644 --- a/moban/data_loaders/manager.py +++ b/moban/data_loaders/manager.py @@ -1,5 +1,7 @@ import os + from lml.plugin import PluginManager + from moban import constants diff --git a/moban/main.py b/moban/main.py index f6d19261..7cccb2e6 100644 --- a/moban/main.py +++ b/moban/main.py @@ -13,8 +13,8 @@ from moban import plugins, reporter, constants, mobanfile, exceptions from moban.utils import merge -from moban.hashstore import HASH_STORE from moban._version import __version__ +from moban.hashstore import HASH_STORE def main(): @@ -116,7 +116,7 @@ def create_parser(): "-v", "--%s" % constants.LABEL_VERSION, action="version", - version="%(prog)s {v}".format(v=__version__) + version="%(prog)s {v}".format(v=__version__), ) return parser From 01d4102b8b4d9c81ccf4c16dc735e71bd18fd72a Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 24 Jan 2019 18:26:57 +0000 Subject: [PATCH 19/38] :fire: remove rnd requirements as moban-handlebars have been updated --- rnd_requirements.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 rnd_requirements.txt diff --git a/rnd_requirements.txt b/rnd_requirements.txt deleted file mode 100644 index 86b965ae..00000000 --- a/rnd_requirements.txt +++ /dev/null @@ -1 +0,0 @@ -https://github.com/moremoban/moban-handlebars/archive/dev.zip From dfbde162e00ba2bfeb3725f05bbedc65fff9bf3f Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 24 Jan 2019 19:36:20 +0000 Subject: [PATCH 20/38] :fire: remove traiing white spaces. black tool does not remove them --- tests/test_docs.py | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/tests/test_docs.py b/tests/test_docs.py index be034ced..08fe2082 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -56,11 +56,11 @@ def test_level_4(self): expected = custom_dedent( """ ========header============ - + world - + shijie - + ========footer============ """ ) @@ -71,13 +71,13 @@ def test_level_5(self): expected = custom_dedent( """ ========header============ - + world - + shijie - + this demonstrates jinja2's include statement - + ========footer============ """ ) @@ -88,13 +88,13 @@ def test_level_6(self): expected = custom_dedent( """ ========header============ - + world2 - + shijie - + this demonstrates jinja2's include statement - + ========footer============ """ ) @@ -105,7 +105,7 @@ def test_level_7(self): expected = custom_dedent( """ Hello, you are in level 7 example - + Hello, you are not in level 7 """ ) @@ -160,11 +160,11 @@ def test_level_13_json(self): expected = custom_dedent( """ ========header============ - + world from child.json - + shijie from parent.yaml - + ========footer============ """ ) @@ -176,11 +176,11 @@ def test_level_13_yaml(self): expected = custom_dedent( """ ========header============ - + world from child.yaml - + shijie from parent.json - + ========footer============ """ ) @@ -192,11 +192,11 @@ def test_level_14_custom(self): expected = custom_dedent( """ ========header============ - + world from child.cusom - + shijie from parent.json - + ========footer============ """ ) From 1856d0a93e0281d65ac593c117929187863bcdb9 Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 24 Jan 2019 19:39:14 +0000 Subject: [PATCH 21/38] :fire: now really kill off python 3.4 test target --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8295c536..16330f5a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,6 @@ python: - 3.7-dev - 3.6 - 3.5 - - 3.4 - 2.7 before_install: - if [[ $TRAVIS_PYTHON_VERSION == "2.6" ]]; then pip install flake8==2.6.2; fi From 0226dd66b1f777b1822369e7453cbcc79e9060c9 Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 25 Jan 2019 08:19:49 +0000 Subject: [PATCH 22/38] :bug: fix UnboundLocalError: local variable 'target' referenced before assignment. fix #178 --- .moban.cd/changelog.yml | 3 ++- CHANGELOG.rst | 4 +++- moban/mobanfile.py | 21 ++++--------------- .../test_command_line_options.py | 9 ++++++++ 4 files changed, 18 insertions(+), 19 deletions(-) diff --git a/.moban.cd/changelog.yml b/.moban.cd/changelog.yml index 9bc95333..72e8288b 100644 --- a/.moban.cd/changelog.yml +++ b/.moban.cd/changelog.yml @@ -6,7 +6,8 @@ releases: details: - "`#107`: Add -v to show current moban version" - "`#164`: support additional data formats" - date: 22-1-2019 + - "`#178`: UnboundLocalError: local variable 'target' referenced before assignment" + date: unreleased version: 0.4.0 - changes: - action: Updated diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d8d21ee2..b7b26858 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,7 +1,7 @@ Change log ================================================================================ -0.4.0 - 22-1-2019 +0.4.0 - unreleased -------------------------------------------------------------------------------- Updated @@ -11,6 +11,8 @@ Updated current moban version #. `#164 `_: support additional data formats +#. `#178 `_: UnboundLocalError: + local variable 'target' referenced before assignment 0.3.9 - 18-1-2019 -------------------------------------------------------------------------------- diff --git a/moban/mobanfile.py b/moban/mobanfile.py index cb43b03e..2e0dd1da 100644 --- a/moban/mobanfile.py +++ b/moban/mobanfile.py @@ -39,20 +39,7 @@ def handle_moban_file_v1(moban_file_configurations, command_line_options): merged_options = None targets = moban_file_configurations.get(constants.LABEL_TARGETS) - try: - target = extract_target(command_line_options) - except Exception as exception: - if targets: - template = command_line_options.get(constants.LABEL_TEMPLATE) - for t in targets: - found_template = template in t.values() - if found_template: - target = [dict(t)] - if not found_template: - # Warn user if template not defined under targets in moban file - reporter.report_template_not_in_moban_file(template) - else: - raise exception + target = extract_target(command_line_options) if constants.LABEL_CONFIG in moban_file_configurations: merged_options = merge( @@ -73,11 +60,11 @@ def handle_moban_file_v1(moban_file_configurations, command_line_options): plugins.ENGINES.register_extensions(extensions) if targets: - # If template specified via CLI flag `-t: - # 1. Only update the specified template - # 2. Do not copy if target: targets = target + # If template specified via CLI flag `-t: + # 1. Only update the specified template + # 2. Do not copy if constants.LABEL_COPY in moban_file_configurations: del moban_file_configurations[constants.LABEL_COPY] number_of_templated_files = handle_targets(merged_options, targets) diff --git a/tests/integration_tests/test_command_line_options.py b/tests/integration_tests/test_command_line_options.py index d8bf7fd1..2eaed307 100644 --- a/tests/integration_tests/test_command_line_options.py +++ b/tests/integration_tests/test_command_line_options.py @@ -147,6 +147,15 @@ def test_single_command(self, fake_template_doer): ], ) + @raises(Exception) + @patch("moban.plugins.BaseEngine.render_to_files") + def test_single_command_with_missing_output(self, fake_template_doer): + test_args = ["moban", "-t", "abc.jj2"] + with patch.object(sys, "argv", test_args): + from moban.main import main + + main() + @patch("moban.plugins.BaseEngine.render_to_files") def test_single_command_with_a_few_options(self, fake_template_doer): test_args = ["moban", "-t", "abc.jj2", "-o", "xyz.output"] From 135ab8ecf4a442f6459a1098b5aa80ab70413a56 Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 25 Jan 2019 19:06:22 +0000 Subject: [PATCH 23/38] :tractor: relocated ~/.moban/repos to appdirs/user_cache_dir()/repos, cross mac, linux and windows --- .moban.cd/changelog.yml | 1 + .moban.cd/moban.yml | 5 +++-- CHANGELOG.rst | 2 ++ requirements.txt | 5 +++-- setup.py | 5 +++-- 5 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.moban.cd/changelog.yml b/.moban.cd/changelog.yml index 9bc95333..4d7541d4 100644 --- a/.moban.cd/changelog.yml +++ b/.moban.cd/changelog.yml @@ -4,6 +4,7 @@ releases: - changes: - action: Updated details: + - "`#174`: Store git cache in XDG_CACHE_DIR" - "`#107`: Add -v to show current moban version" - "`#164`: support additional data formats" date: 22-1-2019 diff --git a/.moban.cd/moban.yml b/.moban.cd/moban.yml index bd2b9205..b9ce2e75 100644 --- a/.moban.cd/moban.yml +++ b/.moban.cd/moban.yml @@ -15,9 +15,10 @@ keywords: - jinja2 - moban dependencies: - - ruamel.yaml + - ruamel.yaml==0.15.86 - jinja2>=2.7.1 - - lml>=0.0.7 + - lml>=0.0.9 + - appdirs==1.4.3 - crayons description: Yet another jinja2 cli command for static text generation scm_host: github.com diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d8d21ee2..6821d500 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,8 @@ Change log Updated ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +#. `#174 `_: Store git cache in + XDG_CACHE_DIR #. `#107 `_: Add -v to show current moban version #. `#164 `_: support additional diff --git a/requirements.txt b/requirements.txt index 2be8834e..38ac0b00 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ -ruamel.yaml +ruamel.yaml==0.15.86 jinja2>=2.7.1 -lml>=0.0.7 +lml>=0.0.9 +appdirs==1.4.3 crayons diff --git a/setup.py b/setup.py index 01d91030..8bc84bf4 100644 --- a/setup.py +++ b/setup.py @@ -46,9 +46,10 @@ ] INSTALL_REQUIRES = [ - 'ruamel.yaml', + 'ruamel.yaml==0.15.86', 'jinja2>=2.7.1', - 'lml>=0.0.7', + 'lml>=0.0.9', + 'appdirs==1.4.3', 'crayons', ] SETUP_COMMANDS = {} From 6833f555178497e6caa84628e630d5091a99193a Mon Sep 17 00:00:00 2001 From: chfw Date: Mon, 28 Jan 2019 18:20:11 +0000 Subject: [PATCH 24/38] :sparkles: feed in the missing changes from this branch --- moban/utils.py | 14 ++++++-------- tests/test_utils.py | 7 +++++++ 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/moban/utils.py b/moban/utils.py index 00b13774..db66e7a0 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -187,14 +187,12 @@ def get_repo_name(repo_url): def get_moban_home(): - home_dir = os.path.expanduser("~") - if os.path.exists(home_dir): - return os.path.join( - home_dir, - constants.MOBAN_DIR_NAME_UNDER_USER_HOME, - constants.MOBAN_REPOS_DIR_NAME, - ) - raise IOError("Failed to find user home directory") + from appdirs import user_cache_dir + home_dir = user_cache_dir(appname=constants.PROGRAM_NAME) + return os.path.join( + home_dir, + constants.MOBAN_REPOS_DIR_NAME, + ) def _remove_dot_git(repo_name): diff --git a/tests/test_utils.py b/tests/test_utils.py index cfa7b12d..4fdd9b2e 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -10,6 +10,7 @@ from moban.utils import ( mkdir_p, get_repo_name, + get_moban_home, write_file_out, file_permissions, get_template_path, @@ -177,3 +178,9 @@ def test_get_repo_name(): actual = [get_repo_name(repo) for repo in repos] expected = ["repo", "repo"] eq_(expected, actual) + + +@patch('appdirs.user_cache_dir', return_value='root') +def test_get_moban_home(_): + actual = get_moban_home() + eq_(os.path.join('root', 'repos'), actual) From 41aa53ad69edee89851c5402085279281dbcdac9 Mon Sep 17 00:00:00 2001 From: chfw Date: Mon, 28 Jan 2019 18:13:05 +0000 Subject: [PATCH 25/38] :sparkles: initial prototype --- moban/utils.py | 18 +++++++----------- tests/test_utils.py | 24 +++++++++++++++--------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/moban/utils.py b/moban/utils.py index db66e7a0..d7ba9e1f 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -133,7 +133,7 @@ def pip_install(packages): def git_clone(repos, submodule=False): - import subprocess + from git import Repo moban_home = get_moban_home() mkdir_p(moban_home) @@ -141,22 +141,18 @@ def git_clone(repos, submodule=False): for repo in repos: repo_name = get_repo_name(repo) local_repo_folder = os.path.join(moban_home, repo_name) - current_working_dir = os.getcwd() if os.path.exists(local_repo_folder): reporter.report_git_pull(repo_name) - os.chdir(local_repo_folder) - subprocess.check_call(["git", "pull"]) + repo = Repo(local_repo_folder) + repo.git.pull() if submodule: - subprocess.check_call(["git", "submodule", "update"]) + repo.git.submodule('update') else: reporter.report_git_clone(repo_name) - os.chdir(moban_home) - subprocess.check_call(["git", "clone", repo, repo_name]) + repo = Repo.clone_from(repo, moban_home) if submodule: - os.chdir(os.path.join(moban_home, repo_name)) - subprocess.check_call(["git", "submodule", "init"]) - subprocess.check_call(["git", "submodule", "update"]) - os.chdir(current_working_dir) + output = repo.git.submodule('update', '--init') + reporter.report_info_message(output) def get_template_path(template_dirs, template): diff --git a/tests/test_utils.py b/tests/test_utils.py index 4fdd9b2e..14f7dd0f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -138,26 +138,32 @@ def test_pip_install(fake_check_all): ) -@patch("subprocess.check_call") -def test_git_clone(fake_check_all): +@patch('os.path.exists', return_value=True) +@patch('git.Repo', autospec=True) +def test_git_clone(fake_repo, _): from moban.utils import git_clone - git_clone(["https://github.com/my/repo", "https://gitlab.com/my/repo"]) - fake_check_all.assert_called_with( - ["git", "clone", "https://gitlab.com/my/repo", "repo"] + repos = ["https://github.com/my/repo", "https://gitlab.com/my/repo"] + git_clone(repos) + fake_repo.assert_called_with( + repos[1] ) + repo = fake_repo.return_value + repo.git.submodule.assert_called_with('update') -@patch("os.chdir") -@patch("subprocess.check_call") -def test_git_clone_with_submodules(fake_check_all, _): +@patch("git.Repo", autospec=True) +def test_git_clone_with_submodules(fake_repo): from moban.utils import git_clone + repo = fake_repo.return_value + git_clone( ["https://github.com/my/repo", "https://gitlab.com/my/repo"], submodule=True, ) - fake_check_all.assert_called_with(["git", "submodule", "update"]) + #fake_check_all.assert_called_with(["git", "submodule", "update"]) + @patch("os.path.exists", return_value=True) From 2b62d8c76e9b09a57e6a104b4b0e469961b8ae66 Mon Sep 17 00:00:00 2001 From: chfw Date: Mon, 28 Jan 2019 22:02:46 +0000 Subject: [PATCH 26/38] :sparkles: replace barebone git with GitPython. fixed #169 --- .moban.cd/moban.yml | 1 + moban/utils.py | 5 ++-- requirements.txt | 1 + setup.py | 1 + tests/test_utils.py | 61 +++++++++++++++++++++++++++++++-------------- 5 files changed, 47 insertions(+), 22 deletions(-) diff --git a/.moban.cd/moban.yml b/.moban.cd/moban.yml index b9ce2e75..7be5db79 100644 --- a/.moban.cd/moban.yml +++ b/.moban.cd/moban.yml @@ -20,5 +20,6 @@ dependencies: - lml>=0.0.9 - appdirs==1.4.3 - crayons + - GitPython==2.1.11 description: Yet another jinja2 cli command for static text generation scm_host: github.com diff --git a/moban/utils.py b/moban/utils.py index d7ba9e1f..ccf61674 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -149,10 +149,9 @@ def git_clone(repos, submodule=False): repo.git.submodule('update') else: reporter.report_git_clone(repo_name) - repo = Repo.clone_from(repo, moban_home) + repo = Repo.clone_from(repo, local_repo_folder) if submodule: - output = repo.git.submodule('update', '--init') - reporter.report_info_message(output) + repo.git.submodule('update', '--init') def get_template_path(template_dirs, template): diff --git a/requirements.txt b/requirements.txt index 38ac0b00..c1dcc49d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ jinja2>=2.7.1 lml>=0.0.9 appdirs==1.4.3 crayons +GitPython==2.1.11 diff --git a/setup.py b/setup.py index 8bc84bf4..4e3f1fde 100644 --- a/setup.py +++ b/setup.py @@ -51,6 +51,7 @@ 'lml>=0.0.9', 'appdirs==1.4.3', 'crayons', + 'GitPython==2.1.11', ] SETUP_COMMANDS = {} diff --git a/tests/test_utils.py b/tests/test_utils.py index 14f7dd0f..9c8a358b 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -139,44 +139,67 @@ def test_pip_install(fake_check_all): @patch('os.path.exists', return_value=True) +@patch('appdirs.user_cache_dir', return_value='root') +@patch('moban.utils.mkdir_p') @patch('git.Repo', autospec=True) -def test_git_clone(fake_repo, _): +def test_git_update(fake_repo, fake_make_dir, *_): from moban.utils import git_clone - repos = ["https://github.com/my/repo", "https://gitlab.com/my/repo"] + repos = ["https://github.com/my/repoA"] git_clone(repos) fake_repo.assert_called_with( - repos[1] + os.path.join('root', 'repos', 'repoA') ) repo = fake_repo.return_value - repo.git.submodule.assert_called_with('update') + eq_(repo.git.pull.called, True) -@patch("git.Repo", autospec=True) -def test_git_clone_with_submodules(fake_repo): +@patch('os.path.exists', return_value=True) +@patch('appdirs.user_cache_dir', return_value='root') +@patch('moban.utils.mkdir_p') +@patch('git.Repo', autospec=True) +def test_git_update_with_submodules(fake_repo, fake_make_dir, *_): from moban.utils import git_clone + repos = ["https://github.com/my/repoA"] + git_clone(repos, submodule=True) + fake_repo.assert_called_with( + os.path.join('root', 'repos', 'repoA') + ) repo = fake_repo.return_value + repo.git.submodule.assert_called_with('update') + - git_clone( - ["https://github.com/my/repo", "https://gitlab.com/my/repo"], - submodule=True, +@patch('os.path.exists', return_value=False) +@patch('appdirs.user_cache_dir', return_value='root') +@patch('moban.utils.mkdir_p') +@patch('git.Repo', autospec=True) +def test_git_checkout_new(fake_repo, fake_make_dir, *_): + from moban.utils import git_clone + + repos = ["https://github.com/my/repoA"] + git_clone(repos) + fake_repo.clone_from.assert_called_with( + repos[0], os.path.join('root', 'repos', 'repoA') ) - #fake_check_all.assert_called_with(["git", "submodule", "update"]) - + repo = fake_repo.return_value + eq_(repo.git.submodule.called, False) -@patch("os.path.exists", return_value=True) -@patch("os.chdir") -@patch("subprocess.check_call") -def test_git_clone_with_existing_repo(fake_check_all, _, __): +@patch('os.path.exists', return_value=False) +@patch('appdirs.user_cache_dir', return_value='root') +@patch('moban.utils.mkdir_p') +@patch('git.Repo', autospec=True) +def test_git_checkout_new_with_submodules(fake_repo, *_): from moban.utils import git_clone - git_clone( - ["https://github.com/my/repo", "https://gitlab.com/my/repo"], - submodule=True, + repos = ["https://github.com/my/repoA"] + git_clone(repos, submodule=True) + fake_repo.clone_from.assert_called_with( + repos[0], os.path.join('root', 'repos', 'repoA') ) - fake_check_all.assert_called_with(["git", "submodule", "update"]) + repo = fake_repo.clone_from.return_value + repo.git.submodule.assert_called_with('update', '--init') def test_get_repo_name(): From 784928282e37c8e04a3e5eafac1bfbc2abdbb439 Mon Sep 17 00:00:00 2001 From: chfw Date: Mon, 28 Jan 2019 22:22:26 +0000 Subject: [PATCH 27/38] :hammer: code refactoring --- moban/utils.py | 10 ++-- tests/test_utils.py | 109 ++++++++++++++++++++------------------------ 2 files changed, 54 insertions(+), 65 deletions(-) diff --git a/moban/utils.py b/moban/utils.py index ccf61674..d63ee429 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -146,12 +146,12 @@ def git_clone(repos, submodule=False): repo = Repo(local_repo_folder) repo.git.pull() if submodule: - repo.git.submodule('update') + repo.git.submodule("update") else: reporter.report_git_clone(repo_name) repo = Repo.clone_from(repo, local_repo_folder) if submodule: - repo.git.submodule('update', '--init') + repo.git.submodule("update", "--init") def get_template_path(template_dirs, template): @@ -183,11 +183,9 @@ def get_repo_name(repo_url): def get_moban_home(): from appdirs import user_cache_dir + home_dir = user_cache_dir(appname=constants.PROGRAM_NAME) - return os.path.join( - home_dir, - constants.MOBAN_REPOS_DIR_NAME, - ) + return os.path.join(home_dir, constants.MOBAN_REPOS_DIR_NAME) def _remove_dot_git(repo_name): diff --git a/tests/test_utils.py b/tests/test_utils.py index 9c8a358b..d125abe9 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -138,68 +138,59 @@ def test_pip_install(fake_check_all): ) -@patch('os.path.exists', return_value=True) @patch('appdirs.user_cache_dir', return_value='root') @patch('moban.utils.mkdir_p') +@patch('os.path.exists') @patch('git.Repo', autospec=True) -def test_git_update(fake_repo, fake_make_dir, *_): - from moban.utils import git_clone - - repos = ["https://github.com/my/repoA"] - git_clone(repos) - fake_repo.assert_called_with( - os.path.join('root', 'repos', 'repoA') - ) - repo = fake_repo.return_value - eq_(repo.git.pull.called, True) - - -@patch('os.path.exists', return_value=True) -@patch('appdirs.user_cache_dir', return_value='root') -@patch('moban.utils.mkdir_p') -@patch('git.Repo', autospec=True) -def test_git_update_with_submodules(fake_repo, fake_make_dir, *_): - from moban.utils import git_clone - - repos = ["https://github.com/my/repoA"] - git_clone(repos, submodule=True) - fake_repo.assert_called_with( - os.path.join('root', 'repos', 'repoA') - ) - repo = fake_repo.return_value - repo.git.submodule.assert_called_with('update') - - -@patch('os.path.exists', return_value=False) -@patch('appdirs.user_cache_dir', return_value='root') -@patch('moban.utils.mkdir_p') -@patch('git.Repo', autospec=True) -def test_git_checkout_new(fake_repo, fake_make_dir, *_): - from moban.utils import git_clone - - repos = ["https://github.com/my/repoA"] - git_clone(repos) - fake_repo.clone_from.assert_called_with( - repos[0], os.path.join('root', 'repos', 'repoA') - ) - repo = fake_repo.return_value - eq_(repo.git.submodule.called, False) - - -@patch('os.path.exists', return_value=False) -@patch('appdirs.user_cache_dir', return_value='root') -@patch('moban.utils.mkdir_p') -@patch('git.Repo', autospec=True) -def test_git_checkout_new_with_submodules(fake_repo, *_): - from moban.utils import git_clone - - repos = ["https://github.com/my/repoA"] - git_clone(repos, submodule=True) - fake_repo.clone_from.assert_called_with( - repos[0], os.path.join('root', 'repos', 'repoA') - ) - repo = fake_repo.clone_from.return_value - repo.git.submodule.assert_called_with('update', '--init') +class TestGitFunctions: + def setUp(self): + self.repos = ["https://github.com/my/repoA"] + + def test_checkout_new(self, fake_repo, local_folder_exists, *_): + from moban.utils import git_clone + + local_folder_exists.return_value = False + git_clone(self.repos) + fake_repo.clone_from.assert_called_with( + self.repos[0], os.path.join('root', 'repos', 'repoA') + ) + repo = fake_repo.return_value + eq_(repo.git.submodule.called, False) + + def test_checkout_new_with_submodules( + self, fake_repo, local_folder_exists, *_): + from moban.utils import git_clone + + local_folder_exists.return_value = False + git_clone(self.repos, submodule=True) + fake_repo.clone_from.assert_called_with( + self.repos[0], os.path.join('root', 'repos', 'repoA') + ) + repo = fake_repo.clone_from.return_value + repo.git.submodule.assert_called_with('update', '--init') + + def test_git_update(self, fake_repo, local_folder_exists, *_): + from moban.utils import git_clone + + local_folder_exists.return_value = True + git_clone(self.repos) + fake_repo.assert_called_with( + os.path.join('root', 'repos', 'repoA') + ) + repo = fake_repo.return_value + eq_(repo.git.pull.called, True) + + def test_git_update_with_submodules( + self, fake_repo, local_folder_exists, *_): + from moban.utils import git_clone + + local_folder_exists.return_value = True + git_clone(self.repos, submodule=True) + fake_repo.assert_called_with( + os.path.join('root', 'repos', 'repoA') + ) + repo = fake_repo.return_value + repo.git.submodule.assert_called_with('update') def test_get_repo_name(): From 7bc51363bbf7fbf10387607c5f0994a99da3e75f Mon Sep 17 00:00:00 2001 From: chfw Date: Mon, 28 Jan 2019 22:33:05 +0000 Subject: [PATCH 28/38] :hammer: code refactor test utils.py --- tests/test_utils.py | 57 +++++++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 31 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index d125abe9..5d18b92c 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -9,6 +9,7 @@ from moban.utils import ( mkdir_p, + git_clone, get_repo_name, get_moban_home, write_file_out, @@ -138,59 +139,53 @@ def test_pip_install(fake_check_all): ) -@patch('appdirs.user_cache_dir', return_value='root') -@patch('moban.utils.mkdir_p') -@patch('os.path.exists') -@patch('git.Repo', autospec=True) +@patch("appdirs.user_cache_dir", return_value="root") +@patch("moban.utils.mkdir_p") +@patch("os.path.exists") +@patch("git.Repo", autospec=True) class TestGitFunctions: def setUp(self): - self.repos = ["https://github.com/my/repoA"] + self.repo_name = "repoA" + self.repo = "https://github.com/my/" + self.repo_name + self.expected_local_repo_path = os.path.join( + "root", "repos", self.repo_name + ) def test_checkout_new(self, fake_repo, local_folder_exists, *_): - from moban.utils import git_clone - local_folder_exists.return_value = False - git_clone(self.repos) + git_clone([self.repo]) fake_repo.clone_from.assert_called_with( - self.repos[0], os.path.join('root', 'repos', 'repoA') + self.repo, self.expected_local_repo_path ) repo = fake_repo.return_value eq_(repo.git.submodule.called, False) def test_checkout_new_with_submodules( - self, fake_repo, local_folder_exists, *_): - from moban.utils import git_clone - + self, fake_repo, local_folder_exists, *_ + ): local_folder_exists.return_value = False - git_clone(self.repos, submodule=True) + git_clone([self.repo], submodule=True) fake_repo.clone_from.assert_called_with( - self.repos[0], os.path.join('root', 'repos', 'repoA') + self.repo, self.expected_local_repo_path ) repo = fake_repo.clone_from.return_value - repo.git.submodule.assert_called_with('update', '--init') + repo.git.submodule.assert_called_with("update", "--init") def test_git_update(self, fake_repo, local_folder_exists, *_): - from moban.utils import git_clone - local_folder_exists.return_value = True - git_clone(self.repos) - fake_repo.assert_called_with( - os.path.join('root', 'repos', 'repoA') - ) + git_clone([self.repo]) + fake_repo.assert_called_with(self.expected_local_repo_path) repo = fake_repo.return_value eq_(repo.git.pull.called, True) def test_git_update_with_submodules( - self, fake_repo, local_folder_exists, *_): - from moban.utils import git_clone - + self, fake_repo, local_folder_exists, *_ + ): local_folder_exists.return_value = True - git_clone(self.repos, submodule=True) - fake_repo.assert_called_with( - os.path.join('root', 'repos', 'repoA') - ) + git_clone([self.repo], submodule=True) + fake_repo.assert_called_with(self.expected_local_repo_path) repo = fake_repo.return_value - repo.git.submodule.assert_called_with('update') + repo.git.submodule.assert_called_with("update") def test_get_repo_name(): @@ -200,7 +195,7 @@ def test_get_repo_name(): eq_(expected, actual) -@patch('appdirs.user_cache_dir', return_value='root') +@patch("appdirs.user_cache_dir", return_value="root") def test_get_moban_home(_): actual = get_moban_home() - eq_(os.path.join('root', 'repos'), actual) + eq_(os.path.join("root", "repos"), actual) From 87a795b97792320b70d8b26b3cf033d9356de45b Mon Sep 17 00:00:00 2001 From: chfw Date: Mon, 28 Jan 2019 22:37:32 +0000 Subject: [PATCH 29/38] :wheelchair: giving better log message --- moban/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/moban/utils.py b/moban/utils.py index d63ee429..ee869971 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -146,11 +146,13 @@ def git_clone(repos, submodule=False): repo = Repo(local_repo_folder) repo.git.pull() if submodule: + reporter.report_info_message("updating submodule") repo.git.submodule("update") else: reporter.report_git_clone(repo_name) repo = Repo.clone_from(repo, local_repo_folder) if submodule: + reporter.report_info_message("checking out submodule") repo.git.submodule("update", "--init") From aad375e1d4872397a214da2116b48c56b2cc4744 Mon Sep 17 00:00:00 2001 From: chfw Date: Mon, 28 Jan 2019 22:39:16 +0000 Subject: [PATCH 30/38] :books: update changelog --- .moban.cd/changelog.yml | 5 ++++- CHANGELOG.rst | 8 +++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.moban.cd/changelog.yml b/.moban.cd/changelog.yml index 6e86c3b1..7f5f97cf 100644 --- a/.moban.cd/changelog.yml +++ b/.moban.cd/changelog.yml @@ -2,12 +2,15 @@ name: moban organisation: moremoban releases: - changes: - - action: Updated + - action: Added details: - "`#174`: Store git cache in XDG_CACHE_DIR" - "`#107`: Add -v to show current moban version" - "`#164`: support additional data formats" + - action: Updated + details: - "`#178`: UnboundLocalError: local variable 'target' referenced before assignment" + - "`#169`: uses GitPython instead of barebone git commands" date: unreleased version: 0.4.0 - changes: diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e8e8a881..d0d72e31 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,7 +4,7 @@ Change log 0.4.0 - unreleased -------------------------------------------------------------------------------- -Updated +Added ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ #. `#174 `_: Store git cache in @@ -13,8 +13,14 @@ Updated current moban version #. `#164 `_: support additional data formats + +Updated +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + #. `#178 `_: UnboundLocalError: local variable 'target' referenced before assignment +#. `#169 `_: uses GitPython + instead of barebone git commands 0.3.9 - 18-1-2019 -------------------------------------------------------------------------------- From fa4b19fc3c1a793c9b728419792e9b9917d1ad90 Mon Sep 17 00:00:00 2001 From: chfw Date: Wed, 30 Jan 2019 18:44:16 +0000 Subject: [PATCH 31/38] :hammer: code refactoring on git clone --- moban/mobanfile.py | 14 +++++--------- moban/utils.py | 12 +++++++----- tests/test_moban_file.py | 19 ++++++++++++++----- tests/test_utils.py | 15 +++++++++++---- 4 files changed, 37 insertions(+), 23 deletions(-) diff --git a/moban/mobanfile.py b/moban/mobanfile.py index 2e0dd1da..affd3687 100644 --- a/moban/mobanfile.py +++ b/moban/mobanfile.py @@ -165,29 +165,25 @@ def extract_target(options): def handle_requires(requires): pypi_pkgs = [] git_repos = [] - git_repos_with_sub = [] for require in requires: if isinstance(require, dict): require_type = require.get(constants.REQUIRE_TYPE, "") if require_type.upper() == constants.GIT_REQUIRE: - submodule_flag = require.get(constants.GIT_HAS_SUBMODULE) - if submodule_flag is True: - git_repos_with_sub.append(require.get(constants.GIT_URL)) - else: - git_repos.append(require.get(constants.GIT_URL)) + git_repos.append(require) elif require_type.upper() == constants.PYPI_REQUIRE: pypi_pkgs.append(require.get(constants.PYPI_PACKAGE_NAME)) else: if is_repo(require): - git_repos.append(require) + git_repos.append({ + constants.GIT_URL: require, + constants.GIT_HAS_SUBMODULE: False + }) else: pypi_pkgs.append(require) if pypi_pkgs: pip_install(pypi_pkgs) if git_repos: git_clone(git_repos) - if git_repos_with_sub: - git_clone(git_repos_with_sub, submodule=True) def is_repo(require): diff --git a/moban/utils.py b/moban/utils.py index ee869971..8953d0c7 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -132,14 +132,16 @@ def pip_install(packages): ) -def git_clone(repos, submodule=False): +def git_clone(requires, submodule=False): from git import Repo moban_home = get_moban_home() mkdir_p(moban_home) - for repo in repos: - repo_name = get_repo_name(repo) + for require in requires: + repo_url = require.get(constants.GIT_URL) + submodule = require.get(constants.GIT_HAS_SUBMODULE, False) + repo_name = get_repo_name(repo_url) local_repo_folder = os.path.join(moban_home, repo_name) if os.path.exists(local_repo_folder): reporter.report_git_pull(repo_name) @@ -149,8 +151,8 @@ def git_clone(repos, submodule=False): reporter.report_info_message("updating submodule") repo.git.submodule("update") else: - reporter.report_git_clone(repo_name) - repo = Repo.clone_from(repo, local_repo_folder) + reporter.report_git_clone(repo_url) + repo = Repo.clone_from(repo_url, local_repo_folder) if submodule: reporter.report_info_message("checking out submodule") repo.git.submodule("update", "--init") diff --git a/tests/test_moban_file.py b/tests/test_moban_file.py index 8c07531f..82d44b02 100644 --- a/tests/test_moban_file.py +++ b/tests/test_moban_file.py @@ -57,8 +57,15 @@ def test_handle_requires_repos(fake_git_clone): repos = ["https://github.com/my/repo", "https://gitlab.com/my/repo"] from moban.mobanfile import handle_requires + expected = [] + for repo in repos: + expected.append({ + "url": repo, + "submodule": False + }) + handle_requires(repos) - fake_git_clone.assert_called_with(repos) + fake_git_clone.assert_called_with(expected) @patch("moban.mobanfile.git_clone") @@ -67,19 +74,21 @@ def test_handle_requires_repos_with_alternative_syntax(fake_git_clone): from moban.mobanfile import handle_requires handle_requires(repos) - fake_git_clone.assert_called_with(["https://github.com/my/repo"]) + fake_git_clone.assert_called_with(repos) +@patch("moban.mobanfile.pip_install") @patch("moban.mobanfile.git_clone") -def test_handle_requires_repos_with_submodule(fake_git_clone): +def test_handle_requires_repos_with_submodule( + fake_git_clone, fake_pip_install): repos = [ {"type": "git", "url": "https://github.com/my/repo", "submodule": True} ] from moban.mobanfile import handle_requires - expected = ["https://github.com/my/repo"] handle_requires(repos) - fake_git_clone.assert_called_with(expected, submodule=True) + fake_git_clone.assert_called_with(repos) + eq_(fake_pip_install.called, False) def test_is_repo(): diff --git a/tests/test_utils.py b/tests/test_utils.py index 5d18b92c..1bdf2284 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -147,13 +147,20 @@ class TestGitFunctions: def setUp(self): self.repo_name = "repoA" self.repo = "https://github.com/my/" + self.repo_name + self.require = { + "url": self.repo + } + self.require_with_submodule = { + "url": self.repo, + "submodule": True + } self.expected_local_repo_path = os.path.join( "root", "repos", self.repo_name ) def test_checkout_new(self, fake_repo, local_folder_exists, *_): local_folder_exists.return_value = False - git_clone([self.repo]) + git_clone([self.require]) fake_repo.clone_from.assert_called_with( self.repo, self.expected_local_repo_path ) @@ -164,7 +171,7 @@ def test_checkout_new_with_submodules( self, fake_repo, local_folder_exists, *_ ): local_folder_exists.return_value = False - git_clone([self.repo], submodule=True) + git_clone([self.require_with_submodule], submodule=True) fake_repo.clone_from.assert_called_with( self.repo, self.expected_local_repo_path ) @@ -173,7 +180,7 @@ def test_checkout_new_with_submodules( def test_git_update(self, fake_repo, local_folder_exists, *_): local_folder_exists.return_value = True - git_clone([self.repo]) + git_clone([self.require]) fake_repo.assert_called_with(self.expected_local_repo_path) repo = fake_repo.return_value eq_(repo.git.pull.called, True) @@ -182,7 +189,7 @@ def test_git_update_with_submodules( self, fake_repo, local_folder_exists, *_ ): local_folder_exists.return_value = True - git_clone([self.repo], submodule=True) + git_clone([self.require_with_submodule], submodule=True) fake_repo.assert_called_with(self.expected_local_repo_path) repo = fake_repo.return_value repo.git.submodule.assert_called_with("update") From 0d4180540f67aaf1119372bdcec748cd53b5fc25 Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 31 Jan 2019 07:12:27 +0000 Subject: [PATCH 32/38] :hammer: code refactoring on git clone functions --- moban/constants.py | 1 + moban/definitions.py | 15 +++++++++++++++ moban/mobanfile.py | 16 +++++++++++----- moban/utils.py | 14 ++++++-------- tests/test_moban_file.py | 18 +++++++++++------- tests/test_utils.py | 16 +++++++--------- 6 files changed, 51 insertions(+), 29 deletions(-) create mode 100644 moban/definitions.py diff --git a/moban/constants.py b/moban/constants.py index d8268fda..9e95dcbe 100644 --- a/moban/constants.py +++ b/moban/constants.py @@ -84,6 +84,7 @@ GIT_REQUIRE = "GIT" GIT_HAS_SUBMODULE = "submodule" GIT_URL = "url" +GIT_BRANCH = "branch" PYPI_REQUIRE = "PYPI" PYPI_PACKAGE_NAME = "name" REQUIRE_TYPE = "type" diff --git a/moban/definitions.py b/moban/definitions.py new file mode 100644 index 00000000..315e4587 --- /dev/null +++ b/moban/definitions.py @@ -0,0 +1,15 @@ +class GitRequire(object): + def __init__(self, git_url=None, branch=None, submodule=False): + self.git_url = git_url + self.submodule = submodule + self.branch = branch + + def __eq__(self, other): + return ( + self.git_url == other.git_url + and self.submodule == other.submodule + and self.branch == other.branch + ) + + def __repr__(self): + return "%s,%s,%s" % (self.git_url, self.branch, self.submodule) diff --git a/moban/mobanfile.py b/moban/mobanfile.py index affd3687..eea481a1 100644 --- a/moban/mobanfile.py +++ b/moban/mobanfile.py @@ -16,6 +16,7 @@ expand_directories, ) from moban.copier import Copier +from moban.definitions import GitRequire try: from urllib.parse import urlparse @@ -169,15 +170,20 @@ def handle_requires(requires): if isinstance(require, dict): require_type = require.get(constants.REQUIRE_TYPE, "") if require_type.upper() == constants.GIT_REQUIRE: - git_repos.append(require) + git_repos.append( + GitRequire( + git_url=require.get(constants.GIT_URL), + branch=require.get(constants.GIT_BRANCH), + submodule=require.get( + constants.GIT_HAS_SUBMODULE, False + ), + ) + ) elif require_type.upper() == constants.PYPI_REQUIRE: pypi_pkgs.append(require.get(constants.PYPI_PACKAGE_NAME)) else: if is_repo(require): - git_repos.append({ - constants.GIT_URL: require, - constants.GIT_HAS_SUBMODULE: False - }) + git_repos.append(GitRequire(require)) else: pypi_pkgs.append(require) if pypi_pkgs: diff --git a/moban/utils.py b/moban/utils.py index 8953d0c7..afdd17a1 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -132,28 +132,26 @@ def pip_install(packages): ) -def git_clone(requires, submodule=False): +def git_clone(requires): from git import Repo moban_home = get_moban_home() mkdir_p(moban_home) for require in requires: - repo_url = require.get(constants.GIT_URL) - submodule = require.get(constants.GIT_HAS_SUBMODULE, False) - repo_name = get_repo_name(repo_url) + repo_name = get_repo_name(require.git_url) local_repo_folder = os.path.join(moban_home, repo_name) if os.path.exists(local_repo_folder): reporter.report_git_pull(repo_name) repo = Repo(local_repo_folder) repo.git.pull() - if submodule: + if require.submodule: reporter.report_info_message("updating submodule") repo.git.submodule("update") else: - reporter.report_git_clone(repo_url) - repo = Repo.clone_from(repo_url, local_repo_folder) - if submodule: + reporter.report_git_clone(require.git_url) + repo = Repo.clone_from(require.git_url, local_repo_folder) + if require.submodule: reporter.report_info_message("checking out submodule") repo.git.submodule("update", "--init") diff --git a/tests/test_moban_file.py b/tests/test_moban_file.py index 82d44b02..78c28070 100644 --- a/tests/test_moban_file.py +++ b/tests/test_moban_file.py @@ -1,6 +1,8 @@ from mock import patch from nose.tools import eq_ +from moban.definitions import GitRequire + class TestFinder: def setUp(self): @@ -59,10 +61,7 @@ def test_handle_requires_repos(fake_git_clone): expected = [] for repo in repos: - expected.append({ - "url": repo, - "submodule": False - }) + expected.append(GitRequire(git_url=repo, submodule=False)) handle_requires(repos) fake_git_clone.assert_called_with(expected) @@ -74,20 +73,25 @@ def test_handle_requires_repos_with_alternative_syntax(fake_git_clone): from moban.mobanfile import handle_requires handle_requires(repos) - fake_git_clone.assert_called_with(repos) + fake_git_clone.assert_called_with( + [GitRequire(git_url="https://github.com/my/repo")] + ) @patch("moban.mobanfile.pip_install") @patch("moban.mobanfile.git_clone") def test_handle_requires_repos_with_submodule( - fake_git_clone, fake_pip_install): + fake_git_clone, fake_pip_install +): repos = [ {"type": "git", "url": "https://github.com/my/repo", "submodule": True} ] from moban.mobanfile import handle_requires handle_requires(repos) - fake_git_clone.assert_called_with(repos) + fake_git_clone.assert_called_with( + [GitRequire(git_url="https://github.com/my/repo", submodule=True)] + ) eq_(fake_pip_install.called, False) diff --git a/tests/test_utils.py b/tests/test_utils.py index 1bdf2284..ddf86bcc 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -20,6 +20,7 @@ strip_off_trailing_new_lines, ) from moban.exceptions import FileNotFound +from moban.definitions import GitRequire def create_file(test_file, permission): @@ -147,13 +148,10 @@ class TestGitFunctions: def setUp(self): self.repo_name = "repoA" self.repo = "https://github.com/my/" + self.repo_name - self.require = { - "url": self.repo - } - self.require_with_submodule = { - "url": self.repo, - "submodule": True - } + self.require = GitRequire(git_url=self.repo) + self.require_with_submodule = GitRequire( + git_url=self.repo, submodule=True + ) self.expected_local_repo_path = os.path.join( "root", "repos", self.repo_name ) @@ -171,7 +169,7 @@ def test_checkout_new_with_submodules( self, fake_repo, local_folder_exists, *_ ): local_folder_exists.return_value = False - git_clone([self.require_with_submodule], submodule=True) + git_clone([self.require_with_submodule]) fake_repo.clone_from.assert_called_with( self.repo, self.expected_local_repo_path ) @@ -189,7 +187,7 @@ def test_git_update_with_submodules( self, fake_repo, local_folder_exists, *_ ): local_folder_exists.return_value = True - git_clone([self.require_with_submodule], submodule=True) + git_clone([self.require_with_submodule]) fake_repo.assert_called_with(self.expected_local_repo_path) repo = fake_repo.return_value repo.git.submodule.assert_called_with("update") From 7fb48f9d5fcd84b2a36a1fb34c87e21c56c8b3d3 Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 31 Jan 2019 07:26:46 +0000 Subject: [PATCH 33/38] :sparkles: checkout default branch only and accept branch parameter in moban file if user choose a different one than the default branch. resolves #172 --- .../README.rst | 1 + moban/definitions.py | 6 +++++ moban/utils.py | 4 +++- tests/test_utils.py | 23 ++++++++++++++++--- 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/docs/level-10-moban-dependency-as-git-repo/README.rst b/docs/level-10-moban-dependency-as-git-repo/README.rst index 6733c3e2..cf5052c3 100644 --- a/docs/level-10-moban-dependency-as-git-repo/README.rst +++ b/docs/level-10-moban-dependency-as-git-repo/README.rst @@ -35,5 +35,6 @@ The alternative syntax is:: - type: git url: https://github.com/your-git-url submodule: true + branch: your_choice_or_default_branch_if_not_specified ... diff --git a/moban/definitions.py b/moban/definitions.py index 315e4587..32c7699e 100644 --- a/moban/definitions.py +++ b/moban/definitions.py @@ -4,6 +4,12 @@ def __init__(self, git_url=None, branch=None, submodule=False): self.submodule = submodule self.branch = branch + def clone_params(self): + clone_params = {"single_branch": True} + if self.branch is not None: + clone_params["branch"] = self.branch + return clone_params + def __eq__(self, other): return ( self.git_url == other.git_url diff --git a/moban/utils.py b/moban/utils.py index afdd17a1..c0aed56c 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -150,7 +150,9 @@ def git_clone(requires): repo.git.submodule("update") else: reporter.report_git_clone(require.git_url) - repo = Repo.clone_from(require.git_url, local_repo_folder) + repo = Repo.clone_from( + require.git_url, local_repo_folder, **require.clone_params() + ) if require.submodule: reporter.report_info_message("checking out submodule") repo.git.submodule("update", "--init") diff --git a/tests/test_utils.py b/tests/test_utils.py index ddf86bcc..d88e0fae 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -152,6 +152,9 @@ def setUp(self): self.require_with_submodule = GitRequire( git_url=self.repo, submodule=True ) + self.require_with_branch = GitRequire( + git_url=self.repo, branch="ghpages" + ) self.expected_local_repo_path = os.path.join( "root", "repos", self.repo_name ) @@ -160,7 +163,7 @@ def test_checkout_new(self, fake_repo, local_folder_exists, *_): local_folder_exists.return_value = False git_clone([self.require]) fake_repo.clone_from.assert_called_with( - self.repo, self.expected_local_repo_path + self.repo, self.expected_local_repo_path, single_branch=True ) repo = fake_repo.return_value eq_(repo.git.submodule.called, False) @@ -171,7 +174,7 @@ def test_checkout_new_with_submodules( local_folder_exists.return_value = False git_clone([self.require_with_submodule]) fake_repo.clone_from.assert_called_with( - self.repo, self.expected_local_repo_path + self.repo, self.expected_local_repo_path, single_branch=True ) repo = fake_repo.clone_from.return_value repo.git.submodule.assert_called_with("update", "--init") @@ -181,7 +184,7 @@ def test_git_update(self, fake_repo, local_folder_exists, *_): git_clone([self.require]) fake_repo.assert_called_with(self.expected_local_repo_path) repo = fake_repo.return_value - eq_(repo.git.pull.called, True) + repo.git.pull.assert_called() def test_git_update_with_submodules( self, fake_repo, local_folder_exists, *_ @@ -192,6 +195,20 @@ def test_git_update_with_submodules( repo = fake_repo.return_value repo.git.submodule.assert_called_with("update") + def test_checkout_new_with_branch( + self, fake_repo, local_folder_exists, *_ + ): + local_folder_exists.return_value = False + git_clone([self.require_with_branch]) + fake_repo.clone_from.assert_called_with( + self.repo, + self.expected_local_repo_path, + branch="ghpages", + single_branch=True, + ) + repo = fake_repo.return_value + eq_(repo.git.submodule.called, False) + def test_get_repo_name(): repos = ["https://github.com/abc/repo", "https://github.com/abc/repo/"] From f2cbee7de89490f362248e48afd755a12507ecfb Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 31 Jan 2019 07:36:36 +0000 Subject: [PATCH 34/38] :green_heart: make pypy build pass as previous used 0.15.87 --- .moban.cd/moban.yml | 2 +- requirements.txt | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.moban.cd/moban.yml b/.moban.cd/moban.yml index 7be5db79..3ded3086 100644 --- a/.moban.cd/moban.yml +++ b/.moban.cd/moban.yml @@ -15,7 +15,7 @@ keywords: - jinja2 - moban dependencies: - - ruamel.yaml==0.15.86 + - ruamel.yaml==0.15.87 - jinja2>=2.7.1 - lml>=0.0.9 - appdirs==1.4.3 diff --git a/requirements.txt b/requirements.txt index c1dcc49d..3c66b4f8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -ruamel.yaml==0.15.86 +ruamel.yaml==0.15.87 jinja2>=2.7.1 lml>=0.0.9 appdirs==1.4.3 diff --git a/setup.py b/setup.py index 4e3f1fde..b8c617a8 100644 --- a/setup.py +++ b/setup.py @@ -46,7 +46,7 @@ ] INSTALL_REQUIRES = [ - 'ruamel.yaml==0.15.86', + 'ruamel.yaml==0.15.87', 'jinja2>=2.7.1', 'lml>=0.0.9', 'appdirs==1.4.3', From bfce80c4483df0e8e9b2ef474c158acc8bbf0985 Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 31 Jan 2019 17:24:14 +0000 Subject: [PATCH 35/38] :sparkles: support git url in ssh syntax. resolves #207 --- .moban.cd/moban.yml | 1 + moban/constants.py | 1 + moban/utils.py | 18 +++++++++++------- requirements.txt | 1 + setup.py | 1 + tests/test_utils.py | 20 ++++++++++++++++++-- 6 files changed, 33 insertions(+), 9 deletions(-) diff --git a/.moban.cd/moban.yml b/.moban.cd/moban.yml index 3ded3086..694cf071 100644 --- a/.moban.cd/moban.yml +++ b/.moban.cd/moban.yml @@ -21,5 +21,6 @@ dependencies: - appdirs==1.4.3 - crayons - GitPython==2.1.11 + - git-url-parse description: Yet another jinja2 cli command for static text generation scm_host: github.com diff --git a/moban/constants.py b/moban/constants.py index 9e95dcbe..7d629797 100644 --- a/moban/constants.py +++ b/moban/constants.py @@ -66,6 +66,7 @@ MESSAGE_DIR_NOT_EXIST = "%s does not exist" MESSAGE_NO_THIRD_PARTY_ENGINE = "No such template support" MESSAGE_FILE_VERSION_NOT_SUPPORTED = "moban file version '%s' is not supported" +MESSAGE_INVALID_GIT_URL = 'An invalid git url: "%s" in mobanfile' # I/O messages # Error handling diff --git a/moban/utils.py b/moban/utils.py index c0aed56c..8eff5b46 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -176,13 +176,17 @@ def get_template_path(template_dirs, template): def get_repo_name(repo_url): - path = repo_url.split("/") - if repo_url.endswith("/"): - repo_name = path[-2] - else: - repo_name = path[-1] - repo_name = _remove_dot_git(repo_name) - return repo_name + import giturlparse + from giturlparse.parser import ParserError + + try: + repo = giturlparse.parse(repo_url) + return repo.name + except ParserError: + reporter.report_error_message( + constants.MESSAGE_INVALID_GIT_URL % repo_url + ) + raise def get_moban_home(): diff --git a/requirements.txt b/requirements.txt index 3c66b4f8..5033a008 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ lml>=0.0.9 appdirs==1.4.3 crayons GitPython==2.1.11 +git-url-parse diff --git a/setup.py b/setup.py index b8c617a8..5d9a474c 100644 --- a/setup.py +++ b/setup.py @@ -52,6 +52,7 @@ 'appdirs==1.4.3', 'crayons', 'GitPython==2.1.11', + 'git-url-parse', ] SETUP_COMMANDS = {} diff --git a/tests/test_utils.py b/tests/test_utils.py index d88e0fae..d61c7f11 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -211,12 +211,28 @@ def test_checkout_new_with_branch( def test_get_repo_name(): - repos = ["https://github.com/abc/repo", "https://github.com/abc/repo/"] + repos = [ + "https://github.com/abc/repo", + "https://github.com/abc/repo.git", + "https://github.com/abc/repo/", + "git@github.com:moremoban/moban.git", + ] actual = [get_repo_name(repo) for repo in repos] - expected = ["repo", "repo"] + expected = ["repo", "repo", "repo", "moban"] eq_(expected, actual) +@patch("moban.reporter.report_error_message") +def test_get_repo_name_can_handle_invalid_url(fake_reporter): + invalid_repo = "invalid" + try: + get_repo_name(invalid_repo) + except Exception: + fake_reporter.assert_called_with( + 'An invalid git url: "invalid" in mobanfile' + ) + + @patch("appdirs.user_cache_dir", return_value="root") def test_get_moban_home(_): actual = get_moban_home() From b7b0c1842f7a73c5c8b6b2002accdbba0f0163fd Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 2 Feb 2019 21:40:35 +0000 Subject: [PATCH 36/38] :egg: :ferris_wheel: release 0.3.10 --- .moban.cd/changelog.yml | 4 ++-- .moban.cd/moban.yml | 6 +++--- CHANGELOG.rst | 2 +- docs/conf.py | 4 ++-- moban/_version.py | 2 +- setup.py | 8 ++++---- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.moban.cd/changelog.yml b/.moban.cd/changelog.yml index 7f5f97cf..342f732e 100644 --- a/.moban.cd/changelog.yml +++ b/.moban.cd/changelog.yml @@ -11,8 +11,8 @@ releases: details: - "`#178`: UnboundLocalError: local variable 'target' referenced before assignment" - "`#169`: uses GitPython instead of barebone git commands" - date: unreleased - version: 0.4.0 + date: 3.2.2019 + version: 0.3.10 - changes: - action: Updated details: diff --git a/.moban.cd/moban.yml b/.moban.cd/moban.yml index 694cf071..96c6d1d9 100644 --- a/.moban.cd/moban.yml +++ b/.moban.cd/moban.yml @@ -3,9 +3,9 @@ organisation: moremoban author: C. W. contact: wangc_2011@hotmail.com license: MIT -version: 0.4.0 -current_version: 0.4.0 -release: 0.3.9 +version: 0.3.10 +current_version: 0.3.10 +release: 0.3.10 branch: master command_line_interface: "moban" entry_point: "moban.main:main" diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d0d72e31..906d98fe 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,7 +1,7 @@ Change log ================================================================================ -0.4.0 - unreleased +0.3.10 - 3.2.2019 -------------------------------------------------------------------------------- Added diff --git a/docs/conf.py b/docs/conf.py index a5d48bef..8cf340f1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -28,9 +28,9 @@ author = u'C. W.' # The short X.Y version -version = u'0.3.9' +version = u'0.3.10' # The full version, including alpha/beta/rc tags -release = u'0.4.0' +release = u'0.3.10' # -- General configuration --------------------------------------------------- diff --git a/moban/_version.py b/moban/_version.py index 44e5b7dc..148c0e0c 100644 --- a/moban/_version.py +++ b/moban/_version.py @@ -1,2 +1,2 @@ -__version__ = "0.4.0" +__version__ = "0.3.10" __author__ = "C. W." diff --git a/setup.py b/setup.py index 5d9a474c..6452c8a8 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ NAME = 'moban' AUTHOR = 'C. W.' -VERSION = '0.4.0' +VERSION = '0.3.10' EMAIL = 'wangc_2011@hotmail.com' LICENSE = 'MIT' ENTRY_POINTS = { @@ -25,7 +25,7 @@ 'Yet another jinja2 cli command for static text generation' ) URL = 'https://github.com/moremoban/moban' -DOWNLOAD_URL = '%s/archive/0.3.9.tar.gz' % URL +DOWNLOAD_URL = '%s/archive/0.3.10.tar.gz' % URL FILES = ['README.rst', 'CONTRIBUTORS.rst', 'CHANGELOG.rst'] KEYWORDS = [ 'python', @@ -63,8 +63,8 @@ # You do not need to read beyond this line PUBLISH_COMMAND = '{0} setup.py sdist bdist_wheel upload -r pypi'.format( sys.executable) -GS_COMMAND = ('gs moban v0.3.9 ' + - "Find 0.3.9 in changelog for more details") +GS_COMMAND = ('gs moban v0.3.10 ' + + "Find 0.3.10 in changelog for more details") NO_GS_MESSAGE = ('Automatic github release is disabled. ' + 'Please install gease to enable it.') UPLOAD_FAILED_MSG = ( From 0c863277bf7feecaa30a9abcf7096f5e733287f7 Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 2 Feb 2019 21:54:28 +0000 Subject: [PATCH 37/38] :muscle: add ayan-b into contributor list --- CONTRIBUTORS.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.rst b/CONTRIBUTORS.rst index 7b29a48f..fd901cbc 100644 --- a/CONTRIBUTORS.rst +++ b/CONTRIBUTORS.rst @@ -3,6 +3,7 @@ Contributors In alphabetical order: +* `Ayan Banerjee `_ * `Charlie Liu `_ * `John Vandenberg `_ * `Joshua Chung `_ From 5a4d41546a99f66b39e7deb3e216c9238fa3b07b Mon Sep 17 00:00:00 2001 From: chfw Date: Sat, 2 Feb 2019 21:56:13 +0000 Subject: [PATCH 38/38] :books: minor update on readme --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 00cdd090..76ecd037 100644 --- a/README.rst +++ b/README.rst @@ -137,7 +137,7 @@ Exit codes -------------------------------------------------------------------------------- By default: -- 0 : no changes +- 0 : no error - 1 : error occured With `--exit-code`: