From 4d046350256fe86ef9001a97c5beb835900c0970 Mon Sep 17 00:00:00 2001 From: chfw Date: Sun, 26 Apr 2020 17:41:38 +0100 Subject: [PATCH 1/2] :books: update readme --- README.rst | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index f8b3b6b9..93392b80 100644 --- a/README.rst +++ b/README.rst @@ -41,15 +41,25 @@ Introduction ================================================================================ **moban** started with bringing the high performance template engine (JINJA2) for web -into static text generation. It has been used in `pyexcel` and `coala` project to keep -documentation consistent across the documentations of individual libraries in the same -organisation. +into static text generation. **moban** can use other python template engine: mako, handlebars, velocity, haml, slim and tornado, can read other data format: json and yaml, and can access both template file and configuration file in any location: zip, git, pypi package, s3, etc. + +It has been used in `pyexcel `_ and +`coala `_ project to keep +documentation consistent across the documentations of individual libraries in the same +organisation. + +And here is a list of other usages: + +1. `Math Sheets `_, generate custom math sheets + in pdf + + Vision ================================================================================ @@ -58,7 +68,7 @@ Any template, any data in any location Support ================================================================================ -If you like moban, please support me on `github `_, +If you like moban, please support me on, `patreon `_ or `bounty source `_ to maintain the project and develop it further. @@ -69,7 +79,7 @@ a little bit more time in coding, documentation and writing interesting extensio Credit ================================================================================ -`jinja2-fsloader` is the key component to enable PyFilesystem2 support in moban +`jinja2-fsloader `_ is the key component to enable PyFilesystem2 support in moban v0.6x. Please show your stars there too! Installation From a1a43ba8e82120d4961e9639c5667906e63884e4 Mon Sep 17 00:00:00 2001 From: jaska Date: Sat, 2 May 2020 00:05:12 +0100 Subject: [PATCH 2/2] Use templates(configuration files) over internet (#373) * tmp do * :tractor: revert the changes in static files * :lipstick: make the implementation proper * :hammer: code refactoring * :green_heart: fix code to pass unit tests * :books: update documentation * :microscope: :books: regression test and documentation * :micrscope: more unit tests * :books: update version and documentation * :fire: remove unused import * :fire: remove unused codes * :books: yaml > allows a v long string with new line. >- does the same but no new line * :eggs: :ferris_wheel: ready to release * :newspaper: add missing test fixture --- .moban.cd/changelog.yml | 6 +++ .moban.cd/moban.yml | 6 +-- CHANGELOG.rst | 8 +++ README.rst | 21 +++++++- docs/README.rst | 3 +- docs/conf.py | 4 +- docs/index.rst | 1 + docs/level-24-files-over-http/.moban.yml | 10 ++++ docs/level-24-files-over-http/README.rst | 54 +++++++++++++++++++ docs/level-24-files-over-http/config.yml | 2 + .../local/demo.txt.jj2 | 1 + moban/_version.py | 2 +- moban/core/hashstore.py | 37 +++++++++---- moban/core/moban_factory.py | 14 +++-- moban/core/mobanfile/__init__.py | 2 + moban/core/utils.py | 30 +++++------ moban/exceptions.py | 8 +++ moban/externals/file_system.py | 11 ++-- moban/main.py | 7 +-- setup.py | 8 +-- tests/requirements.txt | 1 + tests/test_docs.py | 5 ++ tests/test_hash_store.py | 20 ++++++- 23 files changed, 206 insertions(+), 55 deletions(-) create mode 100644 docs/level-24-files-over-http/.moban.yml create mode 100644 docs/level-24-files-over-http/README.rst create mode 100644 docs/level-24-files-over-http/config.yml create mode 100644 docs/level-24-files-over-http/local/demo.txt.jj2 diff --git a/.moban.cd/changelog.yml b/.moban.cd/changelog.yml index db4b4c7b..b0a6f2ba 100644 --- a/.moban.cd/changelog.yml +++ b/.moban.cd/changelog.yml @@ -1,6 +1,12 @@ name: moban organisation: moremoban releases: + - changes: + - action: Added + details: + - "Support for templates and configuration files over HTTP(S) protocol with httpfs! Yepee!" + date: 1.5.2020 + version: 0.7.2 - changes: - action: Fixed details: diff --git a/.moban.cd/moban.yml b/.moban.cd/moban.yml index 6d6cb240..9dfc9851 100644 --- a/.moban.cd/moban.yml +++ b/.moban.cd/moban.yml @@ -4,9 +4,9 @@ organisation: moremoban author: C. W. contact: wangc_2011@hotmail.com license: MIT -version: 0.7.1 -current_version: 0.7.1 -release: 0.7.1 +version: 0.7.2 +current_version: 0.7.2 +release: 0.7.2 branch: master master: index command_line_interface: "moban" diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8f45a05a..c9e63b0a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,14 @@ Change log ================================================================================ +0.7.2 - 1.5.2020 +-------------------------------------------------------------------------------- + +**Added** + +#. Support for templates and configuration files over HTTP(S) protocol with + httpfs! Yepee! + 0.7.1 - 25.04.2020 -------------------------------------------------------------------------------- diff --git a/README.rst b/README.rst index 93392b80..159fea4d 100644 --- a/README.rst +++ b/README.rst @@ -56,6 +56,7 @@ organisation. And here is a list of other usages: +1. `Django Mobans `_, templates for django, docker etc. 1. `Math Sheets `_, generate custom math sheets in pdf @@ -160,7 +161,23 @@ Moban in live action: All use cases are documented `here `_ -Work with files in a git repo +Templates and configuration files over HTTP(S) +================================================================================ + +`httpfs `_ should be installed first. + +.. code-block:: bash + + $ moban -t 'https://raw.githubusercontent.com/moremoban/pypi-mobans/dev/templates/_version.py.jj2'\ + -c 'https://raw.githubusercontent.com/moremoban/pypi-mobans/dev/config/data.yml'\ + -o _version.py + + +In an edge case, if github repo's public url is given, +this github repo shall not have sub repos. This library will fail to +translate sub-repo as url. No magic. + +Templates and configuration files in a git repo ================================================================================ `gitfs2 `_ is optional since v0.7.0 but was @@ -182,7 +199,7 @@ You can do the following with moban: __author__ = "C.W." -Work with files in a python package +Templates and configuration files in a python package ================================================================================ `pypifs `_ is optional since v0.7.0 but diff --git a/docs/README.rst b/docs/README.rst index e5792f2a..483590b5 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -26,6 +26,7 @@ This section covers the use cases for moban. Please check them out individually. #. `Template copying from a zip to a zip`_ #. `Intermeidate targets`_ #. `Mobanfile inheritance`_ +#. `Files over http(s)`_ .. _Jinja2 command line: level-1-jinja2-cli .. _Template inheritance: level-2-template-inheritance @@ -50,4 +51,4 @@ This section covers the use cases for moban. Please check them out individually. .. _Template copying from a zip to a zip: level-21-copy-templates-into-an-alien-file-system .. _Intermeidate targets: level-22-intermediate-targets .. _Mobanfile inheritance: level-23-inherit-organisational-moban-file - +.. _Files over http(s): level-24-files-over-http diff --git a/docs/conf.py b/docs/conf.py index 7f30573d..4331c1ce 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -25,9 +25,9 @@ copyright = '2017-2020 Onni Software Ltd.' author = 'C. W.' # The short X.Y version -version = '0.7.1' +version = '0.7.2' # The full version, including alpha/beta/rc tags -release = '0.7.1' +release = '0.7.2' # -- General configuration --------------------------------------------------- diff --git a/docs/index.rst b/docs/index.rst index b6cd4efe..1d420d57 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -38,6 +38,7 @@ examples folder. level-21-copy-templates-into-an-alien-file-system/README.rst level-22-intermediate-targets/README.rst level-23-inherit-organisational-moban-file/README.rst + level-24-files-over-http/README.rst For more complex use case, please look at `its usage in pyexcel project `_ diff --git a/docs/level-24-files-over-http/.moban.yml b/docs/level-24-files-over-http/.moban.yml new file mode 100644 index 00000000..24b9565a --- /dev/null +++ b/docs/level-24-files-over-http/.moban.yml @@ -0,0 +1,10 @@ +configuration: + template_dir: + - "https://raw.githubusercontent.com/moremoban/pypi-mobans/dev/templates/" + - local + configuration: config.yml + configuration_dir: >- + https://raw.githubusercontent.com/moremoban/pypi-mobans/dev/config/ +targets: + - mytravis.yml: travis.yml.jj2 + - test.txt: demo.txt.jj2 diff --git a/docs/level-24-files-over-http/README.rst b/docs/level-24-files-over-http/README.rst new file mode 100644 index 00000000..86e8c07c --- /dev/null +++ b/docs/level-24-files-over-http/README.rst @@ -0,0 +1,54 @@ +level 24: templates and configuration files over http(s) +================================================================================ + +.. note:: + + You will need to install httpfs + +Why not to take a template off the web? Once a template is written somewhere +by somebody, as long as it is good and useful, it is always to reuse it, +isn't it? DRY principle kicks in. + +Now with mobanfile, it is possible to package up your mobans/templates and +configuration files from a HTTP(S) protocol. + + +Here are the sample file:: + + configuration: + template_dir: + - "https://raw.githubusercontent.com/moremoban/pypi-mobans/dev/templates/" + - local + configuration: config.yml + configuration_dir: "https://raw.githubusercontent.com/moremoban/pypi-mobans/dev/config/" + targets: + - mytravis.yml: travis.yml.jj2 + - test.txt: demo.txt.jj2 + +When you refer to it in configuration section, here is the syntax:: + + configuration: + template_dir: + - "https://raw.githubusercontent.com/moremoban/pypi-mobans/dev/templates/" + +.. warn:: + + The trailing '/' must be there. + + +Maintenance note +-------------------------------------------------------------------------------- + +To the maintainer, in order to eat the dog food. Please checkout pypi-mobans +and run a http server inside local pypi-mobans folder. + +Then update moban's mobanfile to:: + + configuration: + template_dir: + - "http://localhost:8000/templates/" + - "http://localhost:8000/statics/" + - ".moban.d" + +Then run `make update` + diff --git a/docs/level-24-files-over-http/config.yml b/docs/level-24-files-over-http/config.yml new file mode 100644 index 00000000..cdb4f7dd --- /dev/null +++ b/docs/level-24-files-over-http/config.yml @@ -0,0 +1,2 @@ +overrides: data.yml +level24: "files over http protocol" diff --git a/docs/level-24-files-over-http/local/demo.txt.jj2 b/docs/level-24-files-over-http/local/demo.txt.jj2 new file mode 100644 index 00000000..8c478790 --- /dev/null +++ b/docs/level-24-files-over-http/local/demo.txt.jj2 @@ -0,0 +1 @@ +{{name}}: {{level24}} \ No newline at end of file diff --git a/moban/_version.py b/moban/_version.py index 2a8eb359..233ef58a 100644 --- a/moban/_version.py +++ b/moban/_version.py @@ -1,2 +1,2 @@ -__version__ = "0.7.1" +__version__ = "0.7.2" __author__ = "C. W." diff --git a/moban/core/hashstore.py b/moban/core/hashstore.py index aae1d4b5..db428a22 100644 --- a/moban/core/hashstore.py +++ b/moban/core/hashstore.py @@ -1,7 +1,7 @@ import json import hashlib -from moban import constants +from moban import constants, exceptions from moban.externals import file_system @@ -20,21 +20,33 @@ def __init__(self): self.hashes = {} def is_file_changed(self, file_name, file_content, source_template): - changed = self._is_source_updated( + changed, with_permission = self._is_source_updated( file_name, file_content, source_template ) if changed is False: - target_hash = get_file_hash(file_name) + target_hash = get_file_hash( + file_name, with_permission=with_permission + ) if target_hash != self.hashes[file_name]: changed = True return changed def _is_source_updated(self, file_name, file_content, source_template): changed = True - content = _mix( - file_content, oct(file_system.file_permissions(source_template)) - ) + content = file_content + with_permission = True + try: + content = _mix( + file_content, + oct(file_system.file_permissions(source_template)), + ) + except exceptions.NoPermissionsNeeded: + # HttpFs does not have getsyspath + # zip, tar have no permission + # win32 does not work + with_permission = False + pass content_hash = get_hash(content) if file_system.exists(file_name): if file_name in self.hashes: @@ -45,7 +57,7 @@ def _is_source_updated(self, file_name, file_content, source_template): if changed: self.hashes[file_name] = content_hash - return changed + return changed, with_permission def save_hashes(self): with open(self.cache_file, "w") as f: @@ -55,9 +67,16 @@ def save_hashes(self): HASH_STORE = HashStore() -def get_file_hash(afile): +def get_file_hash(afile, with_permission=True): content = file_system.read_bytes(afile) - content = _mix(content, oct(file_system.file_permissions(afile))) + try: + if with_permission: + content = _mix(content, oct(file_system.file_permissions(afile))) + except exceptions.NoPermissionsNeeded: + # HttpFs does not have getsyspath + # zip, tar have no permission + # win32 does not work + pass return get_hash(content) diff --git a/moban/core/moban_factory.py b/moban/core/moban_factory.py index a5081c7f..5b4ad8c5 100644 --- a/moban/core/moban_factory.py +++ b/moban/core/moban_factory.py @@ -1,5 +1,4 @@ import os -import sys import logging from collections import defaultdict @@ -16,7 +15,6 @@ from moban.externals.buffered_writer import BufferedWriter LOG = logging.getLogger(__name__) -PY3_ABOVE = sys.version_info[0] > 2 class MobanFactory(PluginManager): @@ -171,9 +169,15 @@ def apply_template(self, template_abs_path, template, data, output_file): output_file, rendered_content ) if not file_system.is_zip_alike_url(output_file): - file_system.file_permissions_copy( - template_abs_path, output_file - ) + try: + file_system.file_permissions_copy( + template_abs_path, output_file + ) + except exceptions.NoPermissionsNeeded: + # HttpFs does not have getsyspath + # zip, tar have no permission + # win32 does not work + pass return flag except exceptions.FileNotFound: # the template is a string from command line diff --git a/moban/core/mobanfile/__init__.py b/moban/core/mobanfile/__init__.py index 34d3f921..6d875d67 100644 --- a/moban/core/mobanfile/__init__.py +++ b/moban/core/mobanfile/__init__.py @@ -45,6 +45,7 @@ def handle_moban_file_v1(moban_file_configurations, command_line_options): moban_file_configurations[constants.LABEL_CONFIG], ) merged_options = merge(command_line_options, constants.DEFAULT_OPTIONS) + plugins_dirs = merged_options.get(constants.LABEL_PLUGIN_DIRS) if plugins_dirs: handle_plugin_dirs(plugins_dirs) @@ -66,6 +67,7 @@ def handle_moban_file_v1(moban_file_configurations, command_line_options): ) ) ) + extensions = moban_file_configurations.get(constants.LABEL_EXTENSIONS) if extensions: core.ENGINES.register_extensions(extensions) diff --git a/moban/core/utils.py b/moban/core/utils.py index 884e0e19..f00a89fd 100644 --- a/moban/core/utils.py +++ b/moban/core/utils.py @@ -1,32 +1,26 @@ import logging -from moban import constants, exceptions +from moban import constants from moban.externals import file_system LOG = logging.getLogger(__name__) def verify_the_existence_of_directories(dirs): - LOG.debug("Verifying the existence: %s", dirs) if not isinstance(dirs, list): dirs = [dirs] - results = [] - - for directory in dirs: - - if file_system.exists(directory): - results.append(directory) - continue - should_I_ignore = ( + dirs = [ + directory + for directory in dirs + if not ( constants.DEFAULT_CONFIGURATION_DIRNAME in directory or constants.DEFAULT_TEMPLATE_DIRNAME in directory ) - if should_I_ignore: - # ignore - pass - else: - raise exceptions.DirectoryNotFound( - constants.MESSAGE_DIR_NOT_EXIST % directory - ) - return results + ] + if file_system.exists(constants.DEFAULT_CONFIGURATION_DIRNAME): + dirs.append(constants.DEFAULT_CONFIGURATION_DIRNAME) + + if file_system.exists(constants.DEFAULT_TEMPLATE_DIRNAME): + dirs.append(constants.DEFAULT_TEMPLATE_DIRNAME) + return dirs diff --git a/moban/exceptions.py b/moban/exceptions.py index b3907dfc..fcb96e60 100644 --- a/moban/exceptions.py +++ b/moban/exceptions.py @@ -32,3 +32,11 @@ class NoGitCommand(Exception): class UnsupportedPyFS2Protocol(Exception): pass + + +class NoPermissionsNeeded(Exception): + pass + + +class SingleHTTPURLConstraint(Exception): + pass diff --git a/moban/externals/file_system.py b/moban/externals/file_system.py index e890fd47..6639f98c 100644 --- a/moban/externals/file_system.py +++ b/moban/externals/file_system.py @@ -224,12 +224,15 @@ def file_permissions(url): if not exists(url): raise exceptions.FileNotFound(url) if sys.platform == "win32": - return 0 + raise exceptions.NoPermissionsNeeded() elif is_zip_alike_url(url): - return 755 + raise exceptions.NoPermissionsNeeded() else: - unix_path = system_path(url) - return stat.S_IMODE(os.stat(unix_path).st_mode) + try: + unix_path = system_path(url) + return stat.S_IMODE(os.stat(unix_path).st_mode) + except fs.errors.NoSysPath: + raise exceptions.NoPermissionsNeeded() def url_split(url): diff --git a/moban/main.py b/moban/main.py index 379dd8e7..4890a458 100644 --- a/moban/main.py +++ b/moban/main.py @@ -12,6 +12,7 @@ import logging import argparse import logging.config +from io import StringIO from collections import defaultdict from ruamel.yaml import YAML @@ -22,11 +23,6 @@ from moban.externals import reporter, file_system from moban.program_options import OPTIONS -try: - from cStringIO import StringIO -except ImportError: - from io import StringIO - LOG = logging.getLogger() LOG_LEVEL = [logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG] @@ -43,6 +39,7 @@ def main(): options[constants.EXTENSION_DICT] = handle_custom_extensions( options.pop(constants.LABEL_EXTENSION) ) + OPTIONS.update(options) moban_file = options[constants.LABEL_MOBANFILE] if moban_file is None: diff --git a/setup.py b/setup.py index 72af6c1f..5ef79b7a 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ NAME = "moban" AUTHOR = "C. W." -VERSION = "0.7.1" +VERSION = "0.7.2" EMAIL = "wangc_2011@hotmail.com" LICENSE = "MIT" ENTRY_POINTS = { @@ -53,7 +53,7 @@ "General purpose static text generator" ) URL = "https://github.com/moremoban/moban" -DOWNLOAD_URL = "%s/archive/0.7.1.tar.gz" % URL +DOWNLOAD_URL = "%s/archive/0.7.2.tar.gz" % URL FILES = ["README.rst", "CONTRIBUTORS.rst", "CHANGELOG.rst"] KEYWORDS = [ "python", @@ -97,8 +97,8 @@ } # You do not need to read beyond this line PUBLISH_COMMAND = "{0} setup.py sdist bdist_wheel upload -r pypi".format(sys.executable) -GS_COMMAND = ("gs moban v0.7.1 " + - "Find 0.7.1 in changelog for more details") +GS_COMMAND = ("gs moban v0.7.2 " + + "Find 0.7.2 in changelog for more details") NO_GS_MESSAGE = ("Automatic github release is disabled. " + "Please install gease to enable it.") UPLOAD_FAILED_MSG = ( diff --git a/tests/requirements.txt b/tests/requirements.txt index 4979027e..093c2e70 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -15,3 +15,4 @@ jinja2_time pypifs gitfs2 jinja2-python-version>=1.1.2 +httpfs diff --git a/tests/test_docs.py b/tests/test_docs.py index b0c735cb..bbfd821b 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -129,6 +129,11 @@ def test_level_9(self): folder = "level-9-moban-dependency-as-pypi-package" self.run_moban(["moban"], folder, [("test.txt", expected)]) + def test_level_24(self): + expected = "pypi-mobans: files over http protocol" + folder = "level-24-files-over-http" + self.run_moban(["moban"], folder, [("test.txt", expected)]) + def test_level_9_deprecated(self): expected = "pypi-mobans: moban dependency as pypi package" folder = "deprecated-level-9-moban-dependency-as-pypi-package" diff --git a/tests/test_hash_store.py b/tests/test_hash_store.py index 3220fee5..0f098d64 100644 --- a/tests/test_hash_store.py +++ b/tests/test_hash_store.py @@ -1,10 +1,12 @@ import os import sys +from unittest.mock import patch from nose import SkipTest from moban.externals import file_system -from moban.core.hashstore import HashStore +from moban.exceptions import NoPermissionsNeeded +from moban.core.hashstore import HashStore, get_file_hash class TestHashStore: @@ -17,6 +19,7 @@ def setUp(self): "test content".encode("utf-8"), self.source_template, ) + self.file_hash = get_file_hash(self.source_template) def tearDown(self): if os.path.exists(".moban.hashes"): @@ -25,6 +28,21 @@ def tearDown(self): def test_simple_use_case(self): hs = HashStore() flag = hs.is_file_changed(*self.fixture) + + hs.save_hashes() + assert flag is True + + @patch("moban.core.hashstore.file_system.file_permissions") + def test_permission_check_failed(self, fake): + """ + when system permission fails, both source hash and + target hash shall not use permission. + """ + fake.side_effect = [NoPermissionsNeeded()] + hs = HashStore() + flag = hs.is_file_changed(*self.fixture) + + assert hs.hashes["test.out"] != self.file_hash hs.save_hashes() assert flag is True