Use templates(configuration files) over internet

.moban.cd/changelog.yml

@@ -1,6 +1,12 @@
 name: moban
 organisation: moremoban
 releases:
+  - changes:
+      - action: Added
+        details:
+          - "Support for templates and configuration files over HTTP(S) protocol with httpfs! Yepee!"
+    date: 1.5.2020
+    version: 0.7.2
   - changes:
       - action: Fixed
         details:

.moban.cd/moban.yml

@@ -4,9 +4,9 @@ organisation: moremoban
 author: C. W.
 contact: wangc_2011@hotmail.com
 license: MIT
-version: 0.7.1
-current_version: 0.7.1
-release: 0.7.1
+version: 0.7.2
+current_version: 0.7.2
+release: 0.7.2
 branch: master
 master: index
 command_line_interface: "moban"

CHANGELOG.rst

@@ -1,6 +1,14 @@
 Change log
 ================================================================================
 
+0.7.2 - 1.5.2020
+--------------------------------------------------------------------------------
+
+**Added**
+
+#. Support for templates and configuration files over HTTP(S) protocol with
+   httpfs! Yepee!
+
 0.7.1 - 25.04.2020
 --------------------------------------------------------------------------------
 

README.rst

@@ -56,6 +56,7 @@ organisation.
 
 And here is a list of other usages:
 
+1. `Django Mobans <https://github.com/django-mobans>`_, templates for django, docker etc.
 1. `Math Sheets <https://github.com/chfw/math-sheets>`_, generate custom math sheets
    in pdf
 
@@ -160,7 +161,23 @@ Moban in live action:
 All use cases are documented `here <http://moban.readthedocs.org/en/latest/#tutorial>`_
 
 
-Work with files in a git repo
+Templates and configuration files over HTTP(S)
+================================================================================
+
+`httpfs <https://github.com/moremoban/httpfs>`_ should be installed first.
+
+.. code-block:: bash
+
+    $ moban -t 'https://raw.githubusercontent.com/moremoban/pypi-mobans/dev/templates/_version.py.jj2'\
+      -c 'https://raw.githubusercontent.com/moremoban/pypi-mobans/dev/config/data.yml'\
+      -o _version.py
+
+
+In an edge case, if github repo's public url is given,
+this github repo shall not have sub repos. This library will fail to
+translate sub-repo as url. No magic.
+
+Templates and configuration files in a git repo
 ================================================================================
 
 `gitfs2 <https://github.com/moremoban/gitfs2>`_ is optional since v0.7.0 but was
@@ -182,7 +199,7 @@ You can do the following with moban:
     __author__ = "C.W."
 
 
-Work with files in a python package
+Templates and configuration files in a python package
 ================================================================================
 
 `pypifs <https://github.com/moremoban/pypifs>`_ is optional since v0.7.0 but

docs/README.rst

@@ -26,6 +26,7 @@ This section covers the use cases for moban. Please check them out individually.
 #. `Template copying from a zip to a zip`_
 #. `Intermeidate targets`_
 #. `Mobanfile inheritance`_
+#. `Files over http(s)`_
 
 .. _Jinja2 command line: level-1-jinja2-cli
 .. _Template inheritance: level-2-template-inheritance
@@ -50,4 +51,4 @@ This section covers the use cases for moban. Please check them out individually.
 .. _Template copying from a zip to a zip: level-21-copy-templates-into-an-alien-file-system
 .. _Intermeidate targets: level-22-intermediate-targets
 .. _Mobanfile inheritance: level-23-inherit-organisational-moban-file
-
+.. _Files over http(s): level-24-files-over-http

docs/conf.py

@@ -25,9 +25,9 @@
 copyright = '2017-2020 Onni Software Ltd.'
 author = 'C. W.'
 # The short X.Y version
-version = '0.7.1'
+version = '0.7.2'
 # The full version, including alpha/beta/rc tags
-release = '0.7.1'
+release = '0.7.2'
 
 # -- General configuration ---------------------------------------------------
 

docs/index.rst

@@ -38,6 +38,7 @@ examples folder.
    level-21-copy-templates-into-an-alien-file-system/README.rst
    level-22-intermediate-targets/README.rst
    level-23-inherit-organisational-moban-file/README.rst
+   level-24-files-over-http/README.rst
 
 
 For more complex use case, please look at `its usage in pyexcel project <http://pyexcel.readthedocs.io/en/latest/guide.html>`_

docs/level-24-files-over-http/.moban.yml

@@ -0,0 +1,10 @@
+configuration:
+  template_dir:
+    - "https://raw.githubusercontent.com/moremoban/pypi-mobans/dev/templates/"
+    - local
+  configuration: config.yml
+  configuration_dir: >-
+    https://raw.githubusercontent.com/moremoban/pypi-mobans/dev/config/
+targets:
+  - mytravis.yml: travis.yml.jj2
+  - test.txt: demo.txt.jj2

docs/level-24-files-over-http/README.rst

@@ -0,0 +1,54 @@
+level 24: templates and configuration files over http(s)
+================================================================================
+
+.. note::
+
+   You will need to install httpfs
+
+Why not to take a template off the web? Once a template is written somewhere
+by somebody, as long as it is good and useful, it is always to reuse it,
+isn't it? DRY principle kicks in.
+
+Now with mobanfile, it is possible to package up your mobans/templates and
+configuration files from a HTTP(S) protocol.
+
+
+Here are the sample file::
+
+   configuration:
+     template_dir:
+       - "https://raw.githubusercontent.com/moremoban/pypi-mobans/dev/templates/"
+       - local
+     configuration: config.yml
+     configuration_dir: "https://raw.githubusercontent.com/moremoban/pypi-mobans/dev/config/"
+   targets:
+     - mytravis.yml: travis.yml.jj2
+     - test.txt: demo.txt.jj2
+
+When you refer to it in configuration section, here is the syntax::
+
+   configuration:
+     template_dir:
+       - "https://raw.githubusercontent.com/moremoban/pypi-mobans/dev/templates/"
+
+.. warn::
+
+   The trailing '/' must be there.
+
+
+Maintenance note
+--------------------------------------------------------------------------------
+
+To the maintainer, in order to eat the dog food. Please checkout pypi-mobans
+and run a http server inside local pypi-mobans folder.
+
+Then update moban's mobanfile to::
+
+   configuration:
+     template_dir:
+       - "http://localhost:8000/templates/"
+       - "http://localhost:8000/statics/"
+       - ".moban.d"
+
+Then run `make update`
+

docs/level-24-files-over-http/config.yml

@@ -0,0 +1,2 @@
+overrides: data.yml
+level24: "files over http protocol"

docs/level-24-files-over-http/local/demo.txt.jj2

@@ -0,0 +1 @@
+{{name}}: {{level24}}

moban/_version.py

@@ -1,2 +1,2 @@
-__version__ = "0.7.1"
+__version__ = "0.7.2"
 __author__ = "C. W."

moban/core/hashstore.py

@@ -1,7 +1,7 @@
 import json
 import hashlib
 
-from moban import constants
+from moban import constants, exceptions
 from moban.externals import file_system
 
 
@@ -20,21 +20,33 @@ def __init__(self):
             self.hashes = {}
 
     def is_file_changed(self, file_name, file_content, source_template):
-        changed = self._is_source_updated(
+        changed, with_permission = self._is_source_updated(
             file_name, file_content, source_template
         )
 
         if changed is False:
-            target_hash = get_file_hash(file_name)
+            target_hash = get_file_hash(
+                file_name, with_permission=with_permission
+            )
             if target_hash != self.hashes[file_name]:
                 changed = True
         return changed
 
     def _is_source_updated(self, file_name, file_content, source_template):
         changed = True
-        content = _mix(
-            file_content, oct(file_system.file_permissions(source_template))
-        )
+        content = file_content
+        with_permission = True
+        try:
+            content = _mix(
+                file_content,
+                oct(file_system.file_permissions(source_template)),
+            )
+        except exceptions.NoPermissionsNeeded:
+            # HttpFs does not have getsyspath
+            # zip, tar have no permission
+            # win32 does not work
+            with_permission = False
+            pass
         content_hash = get_hash(content)
         if file_system.exists(file_name):
             if file_name in self.hashes:
@@ -45,7 +57,7 @@ def _is_source_updated(self, file_name, file_content, source_template):
         if changed:
             self.hashes[file_name] = content_hash
 
-        return changed
+        return changed, with_permission
 
     def save_hashes(self):
         with open(self.cache_file, "w") as f:
@@ -55,9 +67,16 @@ def save_hashes(self):
 HASH_STORE = HashStore()
 
 
-def get_file_hash(afile):
+def get_file_hash(afile, with_permission=True):
     content = file_system.read_bytes(afile)
-    content = _mix(content, oct(file_system.file_permissions(afile)))
+    try:
+        if with_permission:
+            content = _mix(content, oct(file_system.file_permissions(afile)))
+    except exceptions.NoPermissionsNeeded:
+        # HttpFs does not have getsyspath
+        # zip, tar have no permission
+        # win32 does not work
+        pass
     return get_hash(content)
 
 

moban/core/moban_factory.py

@@ -1,5 +1,4 @@
 import os
-import sys
 import logging
 from collections import defaultdict
 
@@ -16,7 +15,6 @@
 from moban.externals.buffered_writer import BufferedWriter
 
 LOG = logging.getLogger(__name__)
-PY3_ABOVE = sys.version_info[0] > 2
 
 
 class MobanFactory(PluginManager):
@@ -171,9 +169,15 @@ def apply_template(self, template_abs_path, template, data, output_file):
                     output_file, rendered_content
                 )
                 if not file_system.is_zip_alike_url(output_file):
-                    file_system.file_permissions_copy(
-                        template_abs_path, output_file
-                    )
+                    try:
+                        file_system.file_permissions_copy(
+                            template_abs_path, output_file
+                        )
+                    except exceptions.NoPermissionsNeeded:
+                        # HttpFs does not have getsyspath
+                        # zip, tar have no permission
+                        # win32 does not work
+                        pass
             return flag
         except exceptions.FileNotFound:
             # the template is a string from command line

moban/core/mobanfile/__init__.py

@@ -45,6 +45,7 @@ def handle_moban_file_v1(moban_file_configurations, command_line_options):
             moban_file_configurations[constants.LABEL_CONFIG],
         )
     merged_options = merge(command_line_options, constants.DEFAULT_OPTIONS)
+
     plugins_dirs = merged_options.get(constants.LABEL_PLUGIN_DIRS)
     if plugins_dirs:
         handle_plugin_dirs(plugins_dirs)
@@ -66,6 +67,7 @@ def handle_moban_file_v1(moban_file_configurations, command_line_options):
             )
         )
     )
+
     extensions = moban_file_configurations.get(constants.LABEL_EXTENSIONS)
     if extensions:
         core.ENGINES.register_extensions(extensions)

moban/core/utils.py

@@ -1,32 +1,26 @@
 import logging
 
-from moban import constants, exceptions
+from moban import constants
 from moban.externals import file_system
 
 LOG = logging.getLogger(__name__)
 
 
 def verify_the_existence_of_directories(dirs):
-    LOG.debug("Verifying the existence: %s", dirs)
     if not isinstance(dirs, list):
         dirs = [dirs]
 
-    results = []
-
-    for directory in dirs:
-
-        if file_system.exists(directory):
-            results.append(directory)
-            continue
-        should_I_ignore = (
+    dirs = [
+        directory
+        for directory in dirs
+        if not (
             constants.DEFAULT_CONFIGURATION_DIRNAME in directory
             or constants.DEFAULT_TEMPLATE_DIRNAME in directory
         )
-        if should_I_ignore:
-            # ignore
-            pass
-        else:
-            raise exceptions.DirectoryNotFound(
-                constants.MESSAGE_DIR_NOT_EXIST % directory
-            )
-    return results
+    ]
+    if file_system.exists(constants.DEFAULT_CONFIGURATION_DIRNAME):
+        dirs.append(constants.DEFAULT_CONFIGURATION_DIRNAME)
+
+    if file_system.exists(constants.DEFAULT_TEMPLATE_DIRNAME):
+        dirs.append(constants.DEFAULT_TEMPLATE_DIRNAME)
+    return dirs

moban/exceptions.py

@@ -32,3 +32,11 @@ class NoGitCommand(Exception):
 
 class UnsupportedPyFS2Protocol(Exception):
     pass
+
+
+class NoPermissionsNeeded(Exception):
+    pass
+
+
+class SingleHTTPURLConstraint(Exception):
+    pass

moban/externals/file_system.py

@@ -224,12 +224,15 @@ def file_permissions(url):
     if not exists(url):
         raise exceptions.FileNotFound(url)
     if sys.platform == "win32":
-        return 0
+        raise exceptions.NoPermissionsNeeded()
     elif is_zip_alike_url(url):
-        return 755
+        raise exceptions.NoPermissionsNeeded()
     else:
-        unix_path = system_path(url)
-        return stat.S_IMODE(os.stat(unix_path).st_mode)
+        try:
+            unix_path = system_path(url)
+            return stat.S_IMODE(os.stat(unix_path).st_mode)
+        except fs.errors.NoSysPath:
+            raise exceptions.NoPermissionsNeeded()
 
 
 def url_split(url):