diff --git a/.moban.cd/changelog.yml b/.moban.cd/changelog.yml index 2f28e686..342f732e 100644 --- a/.moban.cd/changelog.yml +++ b/.moban.cd/changelog.yml @@ -1,6 +1,18 @@ name: moban organisation: moremoban releases: +- changes: + - action: Added + details: + - "`#174`: Store git cache in XDG_CACHE_DIR" + - "`#107`: Add -v to show current moban version" + - "`#164`: support additional data formats" + - action: Updated + details: + - "`#178`: UnboundLocalError: local variable 'target' referenced before assignment" + - "`#169`: uses GitPython instead of barebone git commands" + date: 3.2.2019 + version: 0.3.10 - changes: - action: Updated details: diff --git a/.moban.cd/moban.yml b/.moban.cd/moban.yml index e049536f..96c6d1d9 100644 --- a/.moban.cd/moban.yml +++ b/.moban.cd/moban.yml @@ -3,9 +3,9 @@ organisation: moremoban author: C. W. contact: wangc_2011@hotmail.com license: MIT -version: 0.3.9 -current_version: 0.3.9 -release: 0.3.9 +version: 0.3.10 +current_version: 0.3.10 +release: 0.3.10 branch: master command_line_interface: "moban" entry_point: "moban.main:main" @@ -15,9 +15,12 @@ keywords: - jinja2 - moban dependencies: - - ruamel.yaml + - ruamel.yaml==0.15.87 - jinja2>=2.7.1 - - lml>=0.0.7 + - lml>=0.0.9 + - appdirs==1.4.3 - crayons + - GitPython==2.1.11 + - git-url-parse description: Yet another jinja2 cli command for static text generation scm_host: github.com diff --git a/.moban.d/travis.yml b/.moban.d/travis.yml index a7ca546e..3408a1ba 100644 --- a/.moban.d/travis.yml +++ b/.moban.d/travis.yml @@ -6,6 +6,5 @@ python: - 3.7-dev - 3.6 - 3.5 - - 3.4 - 2.7 {%endblock%} diff --git a/.travis.yml b/.travis.yml index 8295c536..16330f5a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,6 @@ python: - 3.7-dev - 3.6 - 3.5 - - 3.4 - 2.7 before_install: - if [[ $TRAVIS_PYTHON_VERSION == "2.6" ]]; then pip install flake8==2.6.2; fi diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ac728d82..906d98fe 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,27 @@ Change log ================================================================================ +0.3.10 - 3.2.2019 +-------------------------------------------------------------------------------- + +Added +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +#. `#174 `_: Store git cache in + XDG_CACHE_DIR +#. `#107 `_: Add -v to show + current moban version +#. `#164 `_: support additional + data formats + +Updated +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +#. `#178 `_: UnboundLocalError: + local variable 'target' referenced before assignment +#. `#169 `_: uses GitPython + instead of barebone git commands + 0.3.9 - 18-1-2019 -------------------------------------------------------------------------------- diff --git a/CONTRIBUTORS.rst b/CONTRIBUTORS.rst index 7b29a48f..fd901cbc 100644 --- a/CONTRIBUTORS.rst +++ b/CONTRIBUTORS.rst @@ -3,6 +3,7 @@ Contributors In alphabetical order: +* `Ayan Banerjee `_ * `Charlie Liu `_ * `John Vandenberg `_ * `Joshua Chung `_ diff --git a/README.rst b/README.rst index 00cdd090..76ecd037 100644 --- a/README.rst +++ b/README.rst @@ -137,7 +137,7 @@ Exit codes -------------------------------------------------------------------------------- By default: -- 0 : no changes +- 0 : no error - 1 : error occured With `--exit-code`: diff --git a/docs/README.rst b/docs/README.rst index d3cafab8..960a3562 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -14,6 +14,9 @@ This section covers the use cases for moban. Please check them out individually. #. `Use pypi package as a moban dependency`_ #. `Use git repository as a moban dependency`_ #. `Use handlebars template with moban`_ +#. `Use template engine extensions`_ +#. `Any data overrides any data`_ +#. `Custom data loader`_ .. _Jinja2 command line: level-1-jinja2-cli .. _Template inheritance: level-2-template-inheritance @@ -26,3 +29,6 @@ This section covers the use cases for moban. Please check them out individually. .. _Use pypi package as a moban dependency: level-9-moban-dependency-as-pypi-package .. _Use git repository as a moban dependency: level-10-moban-dependency-as-git-repo .. _Use handlebars template with moban: level-11-use-handlebars +.. _Use template engine extensions: level-12-use-template-engine-extensions +.. _Any data overrides any data: level-13-any-data-override-any-data +.. _Custom data loader: level-14-custom-data-loader diff --git a/docs/conf.py b/docs/conf.py index 2049721a..8cf340f1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -28,9 +28,9 @@ author = u'C. W.' # The short X.Y version -version = u'0.3.9' +version = u'0.3.10' # The full version, including alpha/beta/rc tags -release = u'0.3.9' +release = u'0.3.10' # -- General configuration --------------------------------------------------- diff --git a/docs/level-10-moban-dependency-as-git-repo/README.rst b/docs/level-10-moban-dependency-as-git-repo/README.rst index 6733c3e2..cf5052c3 100644 --- a/docs/level-10-moban-dependency-as-git-repo/README.rst +++ b/docs/level-10-moban-dependency-as-git-repo/README.rst @@ -35,5 +35,6 @@ The alternative syntax is:: - type: git url: https://github.com/your-git-url submodule: true + branch: your_choice_or_default_branch_if_not_specified ... diff --git a/docs/level-13-any-data-override-any-data/.moban.cd/parent.json b/docs/level-13-any-data-override-any-data/.moban.cd/parent.json new file mode 100644 index 00000000..275cdf70 --- /dev/null +++ b/docs/level-13-any-data-override-any-data/.moban.cd/parent.json @@ -0,0 +1,4 @@ +{ + "nihao": "shijie from parent.json", + "hello": "shijie from parent.json" +} diff --git a/docs/level-13-any-data-override-any-data/.moban.cd/parent.yaml b/docs/level-13-any-data-override-any-data/.moban.cd/parent.yaml new file mode 100644 index 00000000..ff67cd90 --- /dev/null +++ b/docs/level-13-any-data-override-any-data/.moban.cd/parent.yaml @@ -0,0 +1,2 @@ +nihao: shijie from parent.yaml +hello: shijie \ No newline at end of file diff --git a/docs/level-13-any-data-override-any-data/.moban.td/base.jj2 b/docs/level-13-any-data-override-any-data/.moban.td/base.jj2 new file mode 100644 index 00000000..1f84d223 --- /dev/null +++ b/docs/level-13-any-data-override-any-data/.moban.td/base.jj2 @@ -0,0 +1,9 @@ +{%block header %} +{%endblock%} + +{{hello}} + +{{nihao}} + +{%block footer %} +{%endblock%} \ No newline at end of file diff --git a/docs/level-13-any-data-override-any-data/README.rst b/docs/level-13-any-data-override-any-data/README.rst new file mode 100644 index 00000000..a6e5d102 --- /dev/null +++ b/docs/level-13-any-data-override-any-data/README.rst @@ -0,0 +1,52 @@ +Level 13: any data override any data +================================================================================ + +It's thought that why shall we constrain ourselves on yaml file format. Along +the development path, json file format was added. What about other file formats? + +By default yaml, json is supported. Due to the new capability `overrides` key +word can override any supported data format:: + + overrides: data.base.json + .... + +or simple use `.json` data instead of `.yaml` data. + +Evaluation +-------------------------------------------------------------------------------- + +Please change directory to `docs/level-13-any-data-override-any-data` directory. + +In this example, `child.yaml` overrides `.moban.cd/parent.json`, here is the +command to launch it: + +.. code-block:: bash + + moban -c child.yaml -t a.template + +'moban.output' is the generated file:: + + ========header============ + + world from child.yaml + + shijie from parent.json + + ========footer============ + + +And we can try `child.json`, which you can guess, overrides `.moban.cd/parent.yaml` + +.. code-block:: bash + + moban -c child.json -t a.template + +'moban.output' is the generated file:: + + ========header============ + + world from child.json + + shijie from parent.yml + + ========footer============ diff --git a/docs/level-13-any-data-override-any-data/a.template b/docs/level-13-any-data-override-any-data/a.template new file mode 100644 index 00000000..44672595 --- /dev/null +++ b/docs/level-13-any-data-override-any-data/a.template @@ -0,0 +1,9 @@ +{%extends 'base.jj2' %} + +{%block header %} +========header============ +{%endblock%} + +{%block footer %} +========footer============ +{%endblock%} diff --git a/docs/level-13-any-data-override-any-data/child.json b/docs/level-13-any-data-override-any-data/child.json new file mode 100644 index 00000000..52d09210 --- /dev/null +++ b/docs/level-13-any-data-override-any-data/child.json @@ -0,0 +1,4 @@ +{ + "overrides": "parent.yaml", + "hello": "world from child.json" +} diff --git a/docs/level-13-any-data-override-any-data/child.yaml b/docs/level-13-any-data-override-any-data/child.yaml new file mode 100644 index 00000000..1a36dfbb --- /dev/null +++ b/docs/level-13-any-data-override-any-data/child.yaml @@ -0,0 +1,2 @@ +overrides: parent.json +hello: world from child.yaml diff --git a/docs/level-14-custom-data-loader/.moban.cd/parent.custom b/docs/level-14-custom-data-loader/.moban.cd/parent.custom new file mode 100644 index 00000000..70d3e256 --- /dev/null +++ b/docs/level-14-custom-data-loader/.moban.cd/parent.custom @@ -0,0 +1,2 @@ +hello,nihao +world from parent.cusom,shijie from parent.custom \ No newline at end of file diff --git a/docs/level-14-custom-data-loader/.moban.cd/parent.json b/docs/level-14-custom-data-loader/.moban.cd/parent.json new file mode 100644 index 00000000..275cdf70 --- /dev/null +++ b/docs/level-14-custom-data-loader/.moban.cd/parent.json @@ -0,0 +1,4 @@ +{ + "nihao": "shijie from parent.json", + "hello": "shijie from parent.json" +} diff --git a/docs/level-14-custom-data-loader/.moban.td/base.jj2 b/docs/level-14-custom-data-loader/.moban.td/base.jj2 new file mode 100644 index 00000000..1f84d223 --- /dev/null +++ b/docs/level-14-custom-data-loader/.moban.td/base.jj2 @@ -0,0 +1,9 @@ +{%block header %} +{%endblock%} + +{{hello}} + +{{nihao}} + +{%block footer %} +{%endblock%} \ No newline at end of file diff --git a/docs/level-14-custom-data-loader/.moban.yml b/docs/level-14-custom-data-loader/.moban.yml new file mode 100644 index 00000000..6058d6c2 --- /dev/null +++ b/docs/level-14-custom-data-loader/.moban.yml @@ -0,0 +1,9 @@ +configuration: + plugin_dir: + - custom-data-loader + template: a.template +targets: + - output: a.output + configuration: child.custom + - output: b.output + configuration: override_custom.yaml diff --git a/docs/level-14-custom-data-loader/README.rst b/docs/level-14-custom-data-loader/README.rst new file mode 100644 index 00000000..7568c33a --- /dev/null +++ b/docs/level-14-custom-data-loader/README.rst @@ -0,0 +1,71 @@ +Level 14: custom data loader +================================================================================ + +Continuing from level 13, `moban` since v0.4.0 allows data loader extension. +Due to the new capability `overrides` key word can override any +data format:: + + overrides: yours.custom + .... + +or simple use `.custom` data instead of `.yaml` data. + +However, you will need to provide a data loader for `.custom` yourselves. + +Evaluation +-------------------------------------------------------------------------------- + +Please change directory to `docs/level-14-custom-data-loader` directory. + + +In this tutorial, a custom data loader was provided to show case its dataloader +extension. Here is the mobanfile:: + + configuration: + plugin_dir: + - custom-data-loader + template: a.template + targets: + - output: a.output + configuration: child.custom + - output: b.output + configuration: override_custom.yaml + +`custom-data-loader` is a directory where custom.py lives. The protocol is +that the custom loader register itself to a file extension and return +a data dictionary confirming mobanfile schema. On call, `moban` will provide +an absolute file name for your loader to work on. + + +Here is the code to do the registration: + +.. code-block:: python + + @PluginInfo(constants.DATA_LOADER_EXTENSION, tags=["custom"]) + + +In order to evaluate, you can simply type:: + + $ moban + $ cat a.output + ========header============ + + world from child.cusom + + shijie from parent.json + + ========footer============ + $ cat b.output + ========header============ + + world from override_custom.yaml + + shijie from parent.custom + + ========footer============ + + +.. warning:: + + Python 2 dictates the existence of __init__.py in the plugin directory. Otheriwse + your plugin won't load diff --git a/docs/level-14-custom-data-loader/a.template b/docs/level-14-custom-data-loader/a.template new file mode 100644 index 00000000..44672595 --- /dev/null +++ b/docs/level-14-custom-data-loader/a.template @@ -0,0 +1,9 @@ +{%extends 'base.jj2' %} + +{%block header %} +========header============ +{%endblock%} + +{%block footer %} +========footer============ +{%endblock%} diff --git a/docs/level-14-custom-data-loader/b.output b/docs/level-14-custom-data-loader/b.output new file mode 100644 index 00000000..48ab2cac --- /dev/null +++ b/docs/level-14-custom-data-loader/b.output @@ -0,0 +1,7 @@ +========header============ + +world from override_custom.yaml + +shijie from parent.custom + +========footer============ diff --git a/docs/level-14-custom-data-loader/child.custom b/docs/level-14-custom-data-loader/child.custom new file mode 100644 index 00000000..25ceefe9 --- /dev/null +++ b/docs/level-14-custom-data-loader/child.custom @@ -0,0 +1,2 @@ +hello,overrides +world from child.cusom,parent.json \ No newline at end of file diff --git a/docs/level-14-custom-data-loader/custom-data-loader/__init__.py b/docs/level-14-custom-data-loader/custom-data-loader/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/docs/level-14-custom-data-loader/custom-data-loader/custom.py b/docs/level-14-custom-data-loader/custom-data-loader/custom.py new file mode 100644 index 00000000..61813cce --- /dev/null +++ b/docs/level-14-custom-data-loader/custom-data-loader/custom.py @@ -0,0 +1,17 @@ +import csv + +from lml.plugin import PluginInfo + +from moban import constants + + +@PluginInfo(constants.DATA_LOADER_EXTENSION, tags=["custom"]) +def open_custom(file_name): + with open(file_name, "r") as data_csv: + csvreader = csv.reader(data_csv) + rows = [] + for row in csvreader: + rows.append(row) + + data = dict(zip(rows[0], rows[1])) + return data diff --git a/docs/level-14-custom-data-loader/override_custom.yaml b/docs/level-14-custom-data-loader/override_custom.yaml new file mode 100644 index 00000000..d24ae6b5 --- /dev/null +++ b/docs/level-14-custom-data-loader/override_custom.yaml @@ -0,0 +1,2 @@ +overrides: parent.custom +hello: world from override_custom.yaml \ No newline at end of file diff --git a/docs/level-3-data-override/README.rst b/docs/level-3-data-override/README.rst index 071b32b4..c163c38a 100644 --- a/docs/level-3-data-override/README.rst +++ b/docs/level-3-data-override/README.rst @@ -21,7 +21,7 @@ command to launch it: moban -c data.yaml -t a.template -'a.output' is the generated file:: +'moban.output' is the generated file:: ========header============ diff --git a/moban/_version.py b/moban/_version.py index bef32dc0..148c0e0c 100644 --- a/moban/_version.py +++ b/moban/_version.py @@ -1,2 +1,2 @@ -__version__ = "0.3.9" +__version__ = "0.3.10" __author__ = "C. W." diff --git a/moban/constants.py b/moban/constants.py index b17eeaca..7d629797 100644 --- a/moban/constants.py +++ b/moban/constants.py @@ -12,6 +12,7 @@ ".%s%s" % (PROGRAM_NAME, ".yaml"), ] DEFAULT_TEMPLATE_TYPE = "jinja2" +DEFAULT_DATA_TYPE = "yaml" # .moban.hashes DEFAULT_MOBAN_CACHE_FILE = ".moban.hashes" @@ -29,6 +30,7 @@ LABEL_OVERRIDES = "overrides" LABEL_MOBANFILE = "mobanfile" LABEL_FORCE = "force" +LABEL_VERSION = "version" DEFAULT_CONFIGURATION_DIRNAME = ".moban.cd" @@ -64,6 +66,7 @@ MESSAGE_DIR_NOT_EXIST = "%s does not exist" MESSAGE_NO_THIRD_PARTY_ENGINE = "No such template support" MESSAGE_FILE_VERSION_NOT_SUPPORTED = "moban file version '%s' is not supported" +MESSAGE_INVALID_GIT_URL = 'An invalid git url: "%s" in mobanfile' # I/O messages # Error handling @@ -82,6 +85,7 @@ GIT_REQUIRE = "GIT" GIT_HAS_SUBMODULE = "submodule" GIT_URL = "url" +GIT_BRANCH = "branch" PYPI_REQUIRE = "PYPI" PYPI_PACKAGE_NAME = "name" REQUIRE_TYPE = "type" @@ -93,6 +97,7 @@ JINJA_GLOBALS_EXTENSION = "jinja_globals" TEMPLATE_ENGINE_EXTENSION = "template_engine" +DATA_LOADER_EXTENSION = "data_loader" LIBRARY_EXTENSION = "library" MOBAN_EXTENSIONS = "^moban_.+$" diff --git a/moban/data_loaders/__init__.py b/moban/data_loaders/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/moban/data_loaders/json_loader.py b/moban/data_loaders/json_loader.py new file mode 100644 index 00000000..30f57268 --- /dev/null +++ b/moban/data_loaders/json_loader.py @@ -0,0 +1,15 @@ +import json + +from lml.plugin import PluginInfo + +from moban import constants + + +@PluginInfo(constants.DATA_LOADER_EXTENSION, tags=["json"]) +def open_json(file_name): + """ + returns json contents as string + """ + with open(file_name, "r") as json_data: + data = json.load(json_data) + return data diff --git a/moban/data_loaders/manager.py b/moban/data_loaders/manager.py new file mode 100644 index 00000000..280499d9 --- /dev/null +++ b/moban/data_loaders/manager.py @@ -0,0 +1,22 @@ +import os + +from lml.plugin import PluginManager + +from moban import constants + + +class AnyDataLoader(PluginManager): + def __init__(self): + super(AnyDataLoader, self).__init__(constants.DATA_LOADER_EXTENSION) + + def get_data(self, file_name): + file_extension = os.path.splitext(file_name)[1] + file_type = file_extension + if file_extension.startswith("."): + file_type = file_type[1:] + + try: + loader_function = self.load_me_now(file_type) + except Exception: + loader_function = self.load_me_now(constants.DEFAULT_DATA_TYPE) + return loader_function(file_name) diff --git a/moban/data_loaders/yaml.py b/moban/data_loaders/yaml.py new file mode 100644 index 00000000..c73feafb --- /dev/null +++ b/moban/data_loaders/yaml.py @@ -0,0 +1,12 @@ +from lml.plugin import PluginInfo +from ruamel.yaml import YAML + +from moban import constants + + +@PluginInfo(constants.DATA_LOADER_EXTENSION, tags=["yaml", "yml"]) +def open_yaml(file_name): + with open(file_name, "r") as data_yaml: + yaml = YAML(typ="rt") + data = yaml.load(data_yaml) + return data diff --git a/moban/definitions.py b/moban/definitions.py new file mode 100644 index 00000000..32c7699e --- /dev/null +++ b/moban/definitions.py @@ -0,0 +1,21 @@ +class GitRequire(object): + def __init__(self, git_url=None, branch=None, submodule=False): + self.git_url = git_url + self.submodule = submodule + self.branch = branch + + def clone_params(self): + clone_params = {"single_branch": True} + if self.branch is not None: + clone_params["branch"] = self.branch + return clone_params + + def __eq__(self, other): + return ( + self.git_url == other.git_url + and self.submodule == other.submodule + and self.branch == other.branch + ) + + def __repr__(self): + return "%s,%s,%s" % (self.git_url, self.branch, self.submodule) diff --git a/moban/main.py b/moban/main.py index f811efa0..7cccb2e6 100644 --- a/moban/main.py +++ b/moban/main.py @@ -11,12 +11,9 @@ import sys import argparse -import moban.reporter as reporter -import moban.constants as constants -import moban.mobanfile as mobanfile -import moban.exceptions as exceptions -from moban import plugins -from moban.utils import merge, open_yaml +from moban import plugins, reporter, constants, mobanfile, exceptions +from moban.utils import merge +from moban._version import __version__ from moban.hashstore import HASH_STORE @@ -115,6 +112,12 @@ def create_parser(): nargs="?", help="string templates", ) + parser.add_argument( + "-v", + "--%s" % constants.LABEL_VERSION, + action="version", + version="%(prog)s {v}".format(v=__version__), + ) return parser @@ -122,7 +125,7 @@ def handle_moban_file(moban_file, options): """ act upon default moban file """ - moban_file_configurations = open_yaml(None, moban_file) + moban_file_configurations = plugins.load_data(None, moban_file) if moban_file_configurations is None: raise exceptions.MobanfileGrammarException( constants.ERROR_INVALID_MOBAN_FILE % moban_file diff --git a/moban/mobanfile.py b/moban/mobanfile.py index cb43b03e..eea481a1 100644 --- a/moban/mobanfile.py +++ b/moban/mobanfile.py @@ -16,6 +16,7 @@ expand_directories, ) from moban.copier import Copier +from moban.definitions import GitRequire try: from urllib.parse import urlparse @@ -39,20 +40,7 @@ def handle_moban_file_v1(moban_file_configurations, command_line_options): merged_options = None targets = moban_file_configurations.get(constants.LABEL_TARGETS) - try: - target = extract_target(command_line_options) - except Exception as exception: - if targets: - template = command_line_options.get(constants.LABEL_TEMPLATE) - for t in targets: - found_template = template in t.values() - if found_template: - target = [dict(t)] - if not found_template: - # Warn user if template not defined under targets in moban file - reporter.report_template_not_in_moban_file(template) - else: - raise exception + target = extract_target(command_line_options) if constants.LABEL_CONFIG in moban_file_configurations: merged_options = merge( @@ -73,11 +61,11 @@ def handle_moban_file_v1(moban_file_configurations, command_line_options): plugins.ENGINES.register_extensions(extensions) if targets: - # If template specified via CLI flag `-t: - # 1. Only update the specified template - # 2. Do not copy if target: targets = target + # If template specified via CLI flag `-t: + # 1. Only update the specified template + # 2. Do not copy if constants.LABEL_COPY in moban_file_configurations: del moban_file_configurations[constants.LABEL_COPY] number_of_templated_files = handle_targets(merged_options, targets) @@ -178,29 +166,30 @@ def extract_target(options): def handle_requires(requires): pypi_pkgs = [] git_repos = [] - git_repos_with_sub = [] for require in requires: if isinstance(require, dict): require_type = require.get(constants.REQUIRE_TYPE, "") if require_type.upper() == constants.GIT_REQUIRE: - submodule_flag = require.get(constants.GIT_HAS_SUBMODULE) - if submodule_flag is True: - git_repos_with_sub.append(require.get(constants.GIT_URL)) - else: - git_repos.append(require.get(constants.GIT_URL)) + git_repos.append( + GitRequire( + git_url=require.get(constants.GIT_URL), + branch=require.get(constants.GIT_BRANCH), + submodule=require.get( + constants.GIT_HAS_SUBMODULE, False + ), + ) + ) elif require_type.upper() == constants.PYPI_REQUIRE: pypi_pkgs.append(require.get(constants.PYPI_PACKAGE_NAME)) else: if is_repo(require): - git_repos.append(require) + git_repos.append(GitRequire(require)) else: pypi_pkgs.append(require) if pypi_pkgs: pip_install(pypi_pkgs) if git_repos: git_clone(git_repos) - if git_repos_with_sub: - git_clone(git_repos_with_sub, submodule=True) def is_repo(require): diff --git a/moban/plugins.py b/moban/plugins.py index 13488cc5..68591f59 100644 --- a/moban/plugins.py +++ b/moban/plugins.py @@ -7,8 +7,13 @@ from moban import utils, constants, exceptions from moban.strategy import Strategy from moban.hashstore import HASH_STORE +from moban.data_loaders.manager import AnyDataLoader -BUILTIN_EXENSIONS = ["moban.jinja2.engine"] +BUILTIN_EXENSIONS = [ + "moban.jinja2.engine", + "moban.data_loaders.yaml", + "moban.data_loaders.json_loader", +] class LibraryManager(PluginManager): @@ -166,6 +171,24 @@ def raise_exception(self, key): LIBRARIES = LibraryManager() ENGINES = EngineFactory() +LOADERS = AnyDataLoader() + + +def load_data(base_dir, file_name): + abs_file_path = utils.search_file(base_dir, file_name) + data = LOADERS.get_data(abs_file_path) + if data is not None: + parent_data = None + if base_dir and constants.LABEL_OVERRIDES in data: + parent_data = load_data( + base_dir, data.pop(constants.LABEL_OVERRIDES) + ) + if parent_data: + return utils.merge(data, parent_data) + else: + return data + else: + return None def expand_template_directories(dirs): @@ -216,13 +239,7 @@ def __init__(self, context_dirs): def get_data(self, file_name): try: - file_extension = os.path.splitext(file_name)[1] - if file_extension == ".json": - data = utils.open_json(self.context_dirs, file_name) - elif file_extension in [".yml", ".yaml"]: - data = utils.open_yaml(self.context_dirs, file_name) - else: - raise exceptions.IncorrectDataInput + data = load_data(self.context_dirs, file_name) utils.merge(data, self.__cached_environ_variables) return data except (IOError, exceptions.IncorrectDataInput) as exception: diff --git a/moban/utils.py b/moban/utils.py index 36de191b..8eff5b46 100644 --- a/moban/utils.py +++ b/moban/utils.py @@ -1,12 +1,9 @@ import os import re import sys -import json import stat import errno -from ruamel.yaml import YAML - import moban.reporter as reporter import moban.constants as constants import moban.exceptions as exceptions @@ -32,38 +29,6 @@ def merge(left, right): return left -def open_yaml(base_dir, file_name): - """ - chained yaml loader - """ - the_yaml_file = search_file(base_dir, file_name) - with open(the_yaml_file, "r") as data_yaml: - yaml = YAML(typ="rt") - data = yaml.load(data_yaml) - if data is not None: - parent_data = None - if base_dir and constants.LABEL_OVERRIDES in data: - parent_data = open_yaml( - base_dir, data.pop(constants.LABEL_OVERRIDES) - ) - if parent_data: - return merge(data, parent_data) - else: - return data - else: - return None - - -def open_json(base_dir, file_name): - """ - returns json contents as string - """ - the_json_file = search_file(base_dir, file_name) - with open(the_json_file, "r") as json_data: - data = json.loads(json_data.read()) - return data - - def search_file(base_dir, file_name): the_file = file_name if not os.path.exists(the_file): @@ -167,31 +132,30 @@ def pip_install(packages): ) -def git_clone(repos, submodule=False): - import subprocess +def git_clone(requires): + from git import Repo moban_home = get_moban_home() mkdir_p(moban_home) - for repo in repos: - repo_name = get_repo_name(repo) + for require in requires: + repo_name = get_repo_name(require.git_url) local_repo_folder = os.path.join(moban_home, repo_name) - current_working_dir = os.getcwd() if os.path.exists(local_repo_folder): reporter.report_git_pull(repo_name) - os.chdir(local_repo_folder) - subprocess.check_call(["git", "pull"]) - if submodule: - subprocess.check_call(["git", "submodule", "update"]) + repo = Repo(local_repo_folder) + repo.git.pull() + if require.submodule: + reporter.report_info_message("updating submodule") + repo.git.submodule("update") else: - reporter.report_git_clone(repo_name) - os.chdir(moban_home) - subprocess.check_call(["git", "clone", repo, repo_name]) - if submodule: - os.chdir(os.path.join(moban_home, repo_name)) - subprocess.check_call(["git", "submodule", "init"]) - subprocess.check_call(["git", "submodule", "update"]) - os.chdir(current_working_dir) + reporter.report_git_clone(require.git_url) + repo = Repo.clone_from( + require.git_url, local_repo_folder, **require.clone_params() + ) + if require.submodule: + reporter.report_info_message("checking out submodule") + repo.git.submodule("update", "--init") def get_template_path(template_dirs, template): @@ -212,24 +176,24 @@ def get_template_path(template_dirs, template): def get_repo_name(repo_url): - path = repo_url.split("/") - if repo_url.endswith("/"): - repo_name = path[-2] - else: - repo_name = path[-1] - repo_name = _remove_dot_git(repo_name) - return repo_name + import giturlparse + from giturlparse.parser import ParserError + + try: + repo = giturlparse.parse(repo_url) + return repo.name + except ParserError: + reporter.report_error_message( + constants.MESSAGE_INVALID_GIT_URL % repo_url + ) + raise def get_moban_home(): - home_dir = os.path.expanduser("~") - if os.path.exists(home_dir): - return os.path.join( - home_dir, - constants.MOBAN_DIR_NAME_UNDER_USER_HOME, - constants.MOBAN_REPOS_DIR_NAME, - ) - raise IOError("Failed to find user home directory") + from appdirs import user_cache_dir + + home_dir = user_cache_dir(appname=constants.PROGRAM_NAME) + return os.path.join(home_dir, constants.MOBAN_REPOS_DIR_NAME) def _remove_dot_git(repo_name): diff --git a/requirements.txt b/requirements.txt index 2be8834e..5033a008 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,7 @@ -ruamel.yaml +ruamel.yaml==0.15.87 jinja2>=2.7.1 -lml>=0.0.7 +lml>=0.0.9 +appdirs==1.4.3 crayons +GitPython==2.1.11 +git-url-parse diff --git a/rnd_requirements.txt b/rnd_requirements.txt deleted file mode 100644 index 86b965ae..00000000 --- a/rnd_requirements.txt +++ /dev/null @@ -1 +0,0 @@ -https://github.com/moremoban/moban-handlebars/archive/dev.zip diff --git a/setup.py b/setup.py index 4967639a..6452c8a8 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ NAME = 'moban' AUTHOR = 'C. W.' -VERSION = '0.3.9' +VERSION = '0.3.10' EMAIL = 'wangc_2011@hotmail.com' LICENSE = 'MIT' ENTRY_POINTS = { @@ -25,7 +25,7 @@ 'Yet another jinja2 cli command for static text generation' ) URL = 'https://github.com/moremoban/moban' -DOWNLOAD_URL = '%s/archive/0.3.9.tar.gz' % URL +DOWNLOAD_URL = '%s/archive/0.3.10.tar.gz' % URL FILES = ['README.rst', 'CONTRIBUTORS.rst', 'CHANGELOG.rst'] KEYWORDS = [ 'python', @@ -46,10 +46,13 @@ ] INSTALL_REQUIRES = [ - 'ruamel.yaml', + 'ruamel.yaml==0.15.87', 'jinja2>=2.7.1', - 'lml>=0.0.7', + 'lml>=0.0.9', + 'appdirs==1.4.3', 'crayons', + 'GitPython==2.1.11', + 'git-url-parse', ] SETUP_COMMANDS = {} @@ -60,8 +63,8 @@ # You do not need to read beyond this line PUBLISH_COMMAND = '{0} setup.py sdist bdist_wheel upload -r pypi'.format( sys.executable) -GS_COMMAND = ('gs moban v0.3.9 ' + - "Find 0.3.9 in changelog for more details") +GS_COMMAND = ('gs moban v0.3.10 ' + + "Find 0.3.10 in changelog for more details") NO_GS_MESSAGE = ('Automatic github release is disabled. ' + 'Please install gease to enable it.') UPLOAD_FAILED_MSG = ( diff --git a/tests/integration_tests/test_command_line_options.py b/tests/integration_tests/test_command_line_options.py index 666fe3f8..2eaed307 100644 --- a/tests/integration_tests/test_command_line_options.py +++ b/tests/integration_tests/test_command_line_options.py @@ -147,6 +147,15 @@ def test_single_command(self, fake_template_doer): ], ) + @raises(Exception) + @patch("moban.plugins.BaseEngine.render_to_files") + def test_single_command_with_missing_output(self, fake_template_doer): + test_args = ["moban", "-t", "abc.jj2"] + with patch.object(sys, "argv", test_args): + from moban.main import main + + main() + @patch("moban.plugins.BaseEngine.render_to_files") def test_single_command_with_a_few_options(self, fake_template_doer): test_args = ["moban", "-t", "abc.jj2", "-o", "xyz.output"] @@ -408,3 +417,12 @@ def test_mako_option(self, fake_template_doer): def tearDown(self): os.unlink(self.config_file) + + +@raises(SystemExit) +def test_version_option(): + test_args = ["moban", "-v"] + with patch.object(sys, "argv", test_args): + from moban.main import main + + main() diff --git a/tests/test_docs.py b/tests/test_docs.py index 99c5bad1..08fe2082 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -1,5 +1,6 @@ import os import sys +from textwrap import dedent from mock import patch from nose.tools import eq_ @@ -7,6 +8,13 @@ from moban.main import main +def custom_dedent(long_texts): + refined = dedent(long_texts) + if refined.startswith("\n"): + refined = refined[1:] + return refined + + class TestTutorial: def setUp(self): self.current = os.getcwd() @@ -17,72 +25,90 @@ def test_level_1(self): self._moban(folder, expected) def test_level_2(self): - expected = """========header============ + expected = custom_dedent( + """ + ========header============ -world + world -========footer============ -""" + ========footer============ + """ + ) folder = "level-2-template-inheritance" self._moban(folder, expected) def test_level_3(self): - expected = """========header============ + expected = custom_dedent( + """ + ========header============ -world + world -shijie + shijie -========footer============ -""" + ========footer============ + """ + ) folder = "level-3-data-override" self._moban(folder, expected) def test_level_4(self): - expected = """========header============ + expected = custom_dedent( + """ + ========header============ -world + world -shijie + shijie -========footer============ -""" + ========footer============ + """ + ) folder = "level-4-single-command" self._raw_moban(["moban"], folder, expected, "a.output") def test_level_5(self): - expected = """========header============ + expected = custom_dedent( + """ + ========header============ -world + world -shijie + shijie -this demonstrates jinja2's include statement + this demonstrates jinja2's include statement -========footer============ -""" + ========footer============ + """ + ) folder = "level-5-custom-configuration" self._raw_moban(["moban"], folder, expected, "a.output") def test_level_6(self): - expected = """========header============ + expected = custom_dedent( + """ + ========header============ -world2 + world2 -shijie + shijie -this demonstrates jinja2's include statement + this demonstrates jinja2's include statement -========footer============ -""" + ========footer============ + """ + ) folder = "level-6-complex-configuration" self._raw_moban(["moban"], folder, expected, "a.output2") def test_level_7(self): - expected = """Hello, you are in level 7 example + expected = custom_dedent( + """ + Hello, you are in level 7 example -Hello, you are not in level 7 -""" + Hello, you are not in level 7 + """ + ) folder = "level-7-use-custom-jinja2-filter-test-n-global" self._raw_moban(["moban"], folder, expected, "test.output") @@ -108,22 +134,76 @@ def test_level_11(self): self._raw_moban(["moban"], folder, expected, "a.output") def test_level_12_a(self): - expected_a = """world -world -world -world -""" + expected_a = custom_dedent( + """ + world + world + world + world + """ + ) folder = "level-12-use-template-engine-extensions" self._raw_moban(["moban"], folder, expected_a, "a.output") def test_level_12_b(self): - expected_b = """142 -42 -142 -""" + expected_b = custom_dedent( + """ + 142 + 42 + 142 + """ + ) folder = "level-12-use-template-engine-extensions" self._raw_moban(["moban"], folder, expected_b, "b.output") + def test_level_13_json(self): + expected = custom_dedent( + """ + ========header============ + + world from child.json + + shijie from parent.yaml + + ========footer============ + """ + ) + folder = "level-13-any-data-override-any-data" + commands = ["moban", "-c", "child.json", "-t", "a.template"] + self._raw_moban(commands, folder, expected, "moban.output") + + def test_level_13_yaml(self): + expected = custom_dedent( + """ + ========header============ + + world from child.yaml + + shijie from parent.json + + ========footer============ + """ + ) + folder = "level-13-any-data-override-any-data" + commands = ["moban", "-c", "child.yaml", "-t", "a.template"] + self._raw_moban(commands, folder, expected, "moban.output") + + def test_level_14_custom(self): + expected = custom_dedent( + """ + ========header============ + + world from child.cusom + + shijie from parent.json + + ========footer============ + """ + ) + folder = "level-14-custom-data-loader" + commands = ["moban"] + self._raw_moban(commands, folder, expected, "a.output") + def test_misc_1(self): expected = "test file\n" diff --git a/tests/test_json_loader.py b/tests/test_json_loader.py new file mode 100644 index 00000000..fc38acd2 --- /dev/null +++ b/tests/test_json_loader.py @@ -0,0 +1,11 @@ +import os + +from nose.tools import eq_ + +from moban.data_loaders.json_loader import open_json + + +def test_open_json(): + content = open_json(os.path.join("tests", "fixtures", "child.json")) + expected = {"key": "hello world", "pass": "ox"} + eq_(expected, content) diff --git a/tests/test_moban_file.py b/tests/test_moban_file.py index 8c07531f..78c28070 100644 --- a/tests/test_moban_file.py +++ b/tests/test_moban_file.py @@ -1,6 +1,8 @@ from mock import patch from nose.tools import eq_ +from moban.definitions import GitRequire + class TestFinder: def setUp(self): @@ -57,8 +59,12 @@ def test_handle_requires_repos(fake_git_clone): repos = ["https://github.com/my/repo", "https://gitlab.com/my/repo"] from moban.mobanfile import handle_requires + expected = [] + for repo in repos: + expected.append(GitRequire(git_url=repo, submodule=False)) + handle_requires(repos) - fake_git_clone.assert_called_with(repos) + fake_git_clone.assert_called_with(expected) @patch("moban.mobanfile.git_clone") @@ -67,19 +73,26 @@ def test_handle_requires_repos_with_alternative_syntax(fake_git_clone): from moban.mobanfile import handle_requires handle_requires(repos) - fake_git_clone.assert_called_with(["https://github.com/my/repo"]) + fake_git_clone.assert_called_with( + [GitRequire(git_url="https://github.com/my/repo")] + ) +@patch("moban.mobanfile.pip_install") @patch("moban.mobanfile.git_clone") -def test_handle_requires_repos_with_submodule(fake_git_clone): +def test_handle_requires_repos_with_submodule( + fake_git_clone, fake_pip_install +): repos = [ {"type": "git", "url": "https://github.com/my/repo", "submodule": True} ] from moban.mobanfile import handle_requires - expected = ["https://github.com/my/repo"] handle_requires(repos) - fake_git_clone.assert_called_with(expected, submodule=True) + fake_git_clone.assert_called_with( + [GitRequire(git_url="https://github.com/my/repo", submodule=True)] + ) + eq_(fake_pip_install.called, False) def test_is_repo(): diff --git a/tests/test_utils.py b/tests/test_utils.py index 609629df..d61c7f11 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -9,8 +9,9 @@ from moban.utils import ( mkdir_p, - open_json, + git_clone, get_repo_name, + get_moban_home, write_file_out, file_permissions, get_template_path, @@ -19,6 +20,7 @@ strip_off_trailing_new_lines, ) from moban.exceptions import FileNotFound +from moban.definitions import GitRequire def create_file(test_file, permission): @@ -138,49 +140,100 @@ def test_pip_install(fake_check_all): ) -@patch("subprocess.check_call") -def test_git_clone(fake_check_all): - from moban.utils import git_clone - - git_clone(["https://github.com/my/repo", "https://gitlab.com/my/repo"]) - fake_check_all.assert_called_with( - ["git", "clone", "https://gitlab.com/my/repo", "repo"] - ) - - -@patch("os.chdir") -@patch("subprocess.check_call") -def test_git_clone_with_submodules(fake_check_all, _): - from moban.utils import git_clone - - git_clone( - ["https://github.com/my/repo", "https://gitlab.com/my/repo"], - submodule=True, - ) - fake_check_all.assert_called_with(["git", "submodule", "update"]) - - -@patch("os.path.exists", return_value=True) -@patch("os.chdir") -@patch("subprocess.check_call") -def test_git_clone_with_existing_repo(fake_check_all, _, __): - from moban.utils import git_clone - - git_clone( - ["https://github.com/my/repo", "https://gitlab.com/my/repo"], - submodule=True, - ) - fake_check_all.assert_called_with(["git", "submodule", "update"]) +@patch("appdirs.user_cache_dir", return_value="root") +@patch("moban.utils.mkdir_p") +@patch("os.path.exists") +@patch("git.Repo", autospec=True) +class TestGitFunctions: + def setUp(self): + self.repo_name = "repoA" + self.repo = "https://github.com/my/" + self.repo_name + self.require = GitRequire(git_url=self.repo) + self.require_with_submodule = GitRequire( + git_url=self.repo, submodule=True + ) + self.require_with_branch = GitRequire( + git_url=self.repo, branch="ghpages" + ) + self.expected_local_repo_path = os.path.join( + "root", "repos", self.repo_name + ) + + def test_checkout_new(self, fake_repo, local_folder_exists, *_): + local_folder_exists.return_value = False + git_clone([self.require]) + fake_repo.clone_from.assert_called_with( + self.repo, self.expected_local_repo_path, single_branch=True + ) + repo = fake_repo.return_value + eq_(repo.git.submodule.called, False) + + def test_checkout_new_with_submodules( + self, fake_repo, local_folder_exists, *_ + ): + local_folder_exists.return_value = False + git_clone([self.require_with_submodule]) + fake_repo.clone_from.assert_called_with( + self.repo, self.expected_local_repo_path, single_branch=True + ) + repo = fake_repo.clone_from.return_value + repo.git.submodule.assert_called_with("update", "--init") + + def test_git_update(self, fake_repo, local_folder_exists, *_): + local_folder_exists.return_value = True + git_clone([self.require]) + fake_repo.assert_called_with(self.expected_local_repo_path) + repo = fake_repo.return_value + repo.git.pull.assert_called() + + def test_git_update_with_submodules( + self, fake_repo, local_folder_exists, *_ + ): + local_folder_exists.return_value = True + git_clone([self.require_with_submodule]) + fake_repo.assert_called_with(self.expected_local_repo_path) + repo = fake_repo.return_value + repo.git.submodule.assert_called_with("update") + + def test_checkout_new_with_branch( + self, fake_repo, local_folder_exists, *_ + ): + local_folder_exists.return_value = False + git_clone([self.require_with_branch]) + fake_repo.clone_from.assert_called_with( + self.repo, + self.expected_local_repo_path, + branch="ghpages", + single_branch=True, + ) + repo = fake_repo.return_value + eq_(repo.git.submodule.called, False) def test_get_repo_name(): - repos = ["https://github.com/abc/repo", "https://github.com/abc/repo/"] + repos = [ + "https://github.com/abc/repo", + "https://github.com/abc/repo.git", + "https://github.com/abc/repo/", + "git@github.com:moremoban/moban.git", + ] actual = [get_repo_name(repo) for repo in repos] - expected = ["repo", "repo"] + expected = ["repo", "repo", "repo", "moban"] eq_(expected, actual) -def test_open_json(): - content = open_json(os.path.join("tests", "fixtures"), "child.json") - expected = {"key": "hello world", "pass": "ox"} - eq_(expected, content) +@patch("moban.reporter.report_error_message") +def test_get_repo_name_can_handle_invalid_url(fake_reporter): + invalid_repo = "invalid" + try: + get_repo_name(invalid_repo) + except Exception: + fake_reporter.assert_called_with( + 'An invalid git url: "invalid" in mobanfile' + ) + + +@patch("appdirs.user_cache_dir", return_value="root") +def test_get_moban_home(_): + actual = get_moban_home() + eq_(os.path.join("root", "repos"), actual) diff --git a/tests/test_yaml_loader.py b/tests/test_yaml_loader.py index 40709c24..8ff4e160 100644 --- a/tests/test_yaml_loader.py +++ b/tests/test_yaml_loader.py @@ -2,34 +2,35 @@ from nose.tools import eq_, raises -from moban.utils import open_yaml +from moban.plugins import load_data +from moban.data_loaders.yaml import open_yaml def test_simple_yaml(): test_file = os.path.join("tests", "fixtures", "simple.yaml") - data = open_yaml(os.path.join("tests", "fixtures"), test_file) + data = open_yaml(test_file) eq_(data, {"simple": "yaml"}) def test_inheritance_yaml(): test_file = os.path.join("tests", "fixtures", "child.yaml") - data = open_yaml(os.path.join("tests", "fixtures", "config"), test_file) + data = load_data(os.path.join("tests", "fixtures", "config"), test_file) eq_(data, {"key": "hello world", "pass": "ox"}) @raises(IOError) def test_exception(): test_file = os.path.join("tests", "fixtures", "orphan.yaml") - open_yaml(os.path.join("tests", "fixtures", "config"), test_file) + load_data(os.path.join("tests", "fixtures", "config"), test_file) @raises(IOError) def test_exception_2(): test_file = os.path.join("tests", "fixtures", "dragon.yaml") - open_yaml(os.path.join("tests", "fixtures", "config"), test_file) + load_data(os.path.join("tests", "fixtures", "config"), test_file) @raises(IOError) def test_exception_3(): test_file = os.path.join("tests", "fixtures", "dragon.yaml") - open_yaml(None, test_file) + load_data(None, test_file)