From f2646d0951ee5a0c28489c68ef04cbf6f89949da Mon Sep 17 00:00:00 2001 From: chfw Date: Thu, 30 Apr 2020 10:18:19 +0100 Subject: [PATCH 01/14] tmp do --- .github/workflows/pythonpublish.yml | 27 +- .travis.yml | 52 +-- CHANGELOG.rst | 583 +--------------------------- docs/conf.py | 86 +--- min_requirements.txt | 11 +- moban/core/hashstore.py | 16 +- moban/core/moban_factory.py | 2 +- moban/core/utils.py | 11 +- mobanfile | 3 +- requirements.txt | 11 +- setup.py | 274 +------------ 11 files changed, 26 insertions(+), 1050 deletions(-) diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml index 9e7ec429..1becba2b 100644 --- a/.github/workflows/pythonpublish.yml +++ b/.github/workflows/pythonpublish.yml @@ -1,26 +1 @@ -name: Upload Python Package - -on: - release: - types: [created] - -jobs: - deploy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - name: Set up Python - uses: actions/setup-python@v1 - with: - python-version: '3.x' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine - - name: Build and publish - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - python setup.py sdist bdist_wheel - twine upload dist/* +404: Not Found \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index ad53eb84..1becba2b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,51 +1 @@ -sudo: false -dist: xenial -language: python -notifications: - email: false -python: - - 3.7 - - 3.6 - - 3.8 -env: - - MINREQ=0 - - MINREQ=1 - -stages: - - lint - - moban - - test - - -.lint: &lint - git: - submodules: false - python: 3.6 - stage: lint - script: make install_test format git-diff-check lint - -.moban: &moban - python: 3.6 - stage: moban - install: pip install moban>=0.0.4 gitfs2 pypifs - script: make update git-diff-check - -jobs: - include: - - *moban - - *lint - -stage: test - -before_install: - - if [[ -f min_requirements.txt && "$MINREQ" -eq 1 ]]; then - mv min_requirements.txt requirements.txt ; - fi - - test ! -f rnd_requirements.txt || - pip install --no-deps -r rnd_requirements.txt - - test ! -f rnd_requirements.txt || pip install -r rnd_requirements.txt ; - - pip install -r tests/requirements.txt -script: - - make test -after_success: - codecov +404: Not Found \ No newline at end of file diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8f45a05a..1becba2b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,582 +1 @@ -Change log -================================================================================ - -0.7.1 - 25.04.2020 --------------------------------------------------------------------------------- - -**Fixed** - -#. `#365 `_: regression was - introduced by v0.6.5. If you uses mobanfile as data configuration file, you - are very likely to have this show stopper. Please upgrade to this version. - -0.7.0 - 18.01.2020 --------------------------------------------------------------------------------- - -**Removed** - -#. `#360 `_: make gitfs2 and - pypifs optional. -#. `#303 `_: python 2.7 support - is dropped. - -**Updated** - -#. `#360 `_: show friendlier - error when unknown protocol exception was raised. - -0.6.8 - 7.12.2019 --------------------------------------------------------------------------------- - -**Updated** - -#. since version 0.5.0, when rendering a single file or string, moban would - report 'Templated 1 of 0 files', which should have been 'Templated 1 file.' - -**Removed** - -#. python 3.4 support is gone because colorama requires Python '>=2.7, !=3.0.*, - !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*' - -0.6.7 - 1.12.2019 --------------------------------------------------------------------------------- - -**Updated** - -#. no verbose for error, -v for warning, -vv for warning+info, -vvv for - warning+info+debug -#. `#351 `_, show template plugin - name, i.e. 'copying' for copy instead of 'templating' - -**Removed** - -#. Message: 'Warning: Attempting to use environment vars as data...' became - warning log -#. Message: 'Warning: Both data.yml and /.../.moban.cd/data.yml does not exist' - became warning log -#. with -v, you would see them in such a situation - -0.6.6 - 10.11.2019 --------------------------------------------------------------------------------- - -**Added** - -#. support moban file inheritance. one base moban file and child repos can - inherit and override - -0.6.5 - 13.10.2019 --------------------------------------------------------------------------------- - -**Added** - -#. `#335 `_: support intermediate - targets in moban file - -0.6.4 - 4.10.2019 --------------------------------------------------------------------------------- - -**Updated** - -#. Command options have been grouped. --template_type became --template-type -#. Increment gitfs2 to version 0.0.2. `gitfs#4 - `_ - -0.6.3 - 25.09.2019 --------------------------------------------------------------------------------- - -**Added** - -#. `#260 `_: jinja-cli parity: - support command line pipe stream. - -0.6.2 - 15.09.2019 --------------------------------------------------------------------------------- - -**Added** - -#. `#322 `_: Implicit targets - with template extensions default to copy -#. `#257 `_: '-e' to load - extensions for template engines, i.e. jinja2 -#. `#333 `_: command line - template fails with version 0.6.1 - -0.6.1 - 10.09.2019 --------------------------------------------------------------------------------- - -**Fixed** - -#. `#328 `_: update backward - compatibility - -0.6.0 - 10.09.2019 --------------------------------------------------------------------------------- - -**Added** - -#. `#205 `_: support - `pyFilesystem2 `_ -#. `#185 `_: -v will enable moban - application logging for development. And -V is for version. -#. `#325 `_: -vv show debug trace -#. `#126 `_: Allow mobanfile to - include data from arbitrary config files -#. `#256 `_: jinja2-cli parity: - '-d hello=world' to define custom variable on cli - -**Updated** - -#. `#275 `_: fix moban 0.4.5 test - failures on openSUSE Tumbleweed - -0.5.0 - 14.07.2019 --------------------------------------------------------------------------------- - -**Updated** - -#. `#277 `_: Restored dependency - git-url-parse, replacing incompatible giturlparse which was used during moban - 0.4.x -#. `#281 `_: Fixed unicode - support on Python 2.7 -#. `#274 `_: Updated ruamel.yaml - dependency pins to restore support for Python 3.4, and prevent installation - of versions that can not be installed on Python 3.7 -#. `#285 `_: Fixed CI testing of - minimum requirements -#. `#271 `_: Fixed repository - caching bug preventing branch switching -#. `#292 `_: Reformatted YAML - files according to yamllint rules -#. `#291 `_: Fixed filename typos - in README -#. `#280 `_: Added CI to ensure - repository is in sync with upstream -#. `#280 `_: sync setup.py from - pypi-mobans - -0.4.5 - 07.07.2019 --------------------------------------------------------------------------------- - -**Updated** - -#. `#271 `_: support git branch - change in later run. - -0.4.4 - 26.05.2019 --------------------------------------------------------------------------------- - -**Updated** - -#. `#265 `_: Use simple `read - binary` to read instead of encoding - -0.4.3 - 16.03.2019 --------------------------------------------------------------------------------- - -**Removed** - -#. `#253 `_: symbolic link in - regression pack causes python setup.py to do recursive include - -**Added** - -#. `#209 `_: Alert moban user - when `git` is not available and is used. - -**Updated** - -#. `#261 `_: since moban group - template files per template type, this fill use first come first register to - order moban group - -0.4.2 - 08.03.2019 --------------------------------------------------------------------------------- - -**Added** - -#. `#234 `_: Define template - parameters on the fly inside `targets` section -#. `#62 `_: select a group target - to run - -**Updated** - -#. `#180 `_: No longer two - statistics will be shown in v0.4.x. legacy copy targets are injected into a - normal targets. cli target is made a clear priority. -#. `#244 `_: version 0.4.2 is - first version which would work perfectly on windows since 17 Nov 2018. Note - that: file permissions are not used on windows. Why the date? because - samefile is not avaiable on windows, causing unit tests to fail hence it lead - to my conclusion that moban version between 17 Nov 2018 and March 2019 wont - work well on Windows. - -0.4.1 - 28.02.2019 --------------------------------------------------------------------------------- - -**Added** - -#. `#235 `_: user defined - template types so that custom file extensions, template configurations can be - controlled by moban user -#. `#232 `_: the package - dependencies have been fine tuning to lower versions, most of them are dated - back to 2017. - -0.4.0 - 20.02.2019 --------------------------------------------------------------------------------- - -**Added** - -#. `#165 `_: Copy as plugins - -**Updated** - -#. `#219 `_: git clone depth set - to 2 -#. `#186 `_: lowest dependecy on - ruamel.yaml is 0.15.5, Jun 2017 - -0.3.10 - 03.02.2019 --------------------------------------------------------------------------------- - -**Added** - -#. `#174 `_: Store git cache in - XDG_CACHE_DIR -#. `#107 `_: Add -v to show - current moban version -#. `#164 `_: support additional - data formats - -**Updated** - -#. `#178 `_: UnboundLocalError: - local variable 'target' referenced before assignment -#. `#169 `_: uses GitPython - instead of barebone git commands - -0.3.9 - 18-1-2019 --------------------------------------------------------------------------------- - -**Updated** - -#. `#90 `_: allow adding extra - jinja2 extensions. `jinja2.ext.do`, `jinja2.ext.loopcontrols` are included by - default. what's more, any other template enigne are eligible for extension - additions. -#. `#158 `_: Empty file - base_engine.py is finally removed - -0.3.8 - 12-1-2019 --------------------------------------------------------------------------------- - -**Updated** - -#. `#141 `_: disable file - permissions copy feature and not to check file permission changes on windows. -#. `#154 `_: introduce first ever - positional argument for string base template. -#. `#157 `_: the exit code - behavior changed. for backward compactibility please use --exit-code. - Otherwise, moban will not tell if there is any changes. - -0.3.7 - 6-1-2019 --------------------------------------------------------------------------------- - -**Updated** - -#. `#146 `_: added a low-setup - usage mode via environment variables to moban -#. `#148 `_: include test related - files in the package for package validation when distributing via linux - system, i.e. OpenSuse - -0.3.6 - 30-12-2018 --------------------------------------------------------------------------------- - -**Updated** - -#. `#143 `_: moban shall report - permission error and continue the rest of the copying task. -#. `#122 `_: Since 0.3.6, moban - is tested on windows and macos too, using azure build pipelines. It is - already tested extensively on travis-ci on linux os. - -0.3.5 - 10-12-2018 --------------------------------------------------------------------------------- - -**Updated** - -#. `#37 `_: moban will report line - number where the value is empty and the name of mobanfile. Switch from pyyaml - to ruamel.yaml. - -0.3.4.1 - 28-11-2018 --------------------------------------------------------------------------------- - -**Updated** - -#. `#137 `_: missing - contributors.rst file - -0.3.4 - 18-11-2018 --------------------------------------------------------------------------------- - -**Added** - -#. global variables to store the target and template file names in the jinja2 - engine -#. moban-handlebars is tested to work well with this version and above - -**Updated** - -#. Template engine interface has been clarified and documented - -0.3.3 - 05-11-2018 --------------------------------------------------------------------------------- - -**Added** - -#. alternative and expanded syntax for requires, so as to accomendate github - submodule recursive - -0.3.2 - 04-11-2018 --------------------------------------------------------------------------------- - -**Added** - -#. configuration dirs may be located by `requires`, i.e. configuration files may - be in a python package or git repository. - -0.3.1 - 02-11-2018 --------------------------------------------------------------------------------- - -**Added** - -#. `#97 `_: requires will clone a - repo if given. Note: only github, gitlab, bitbucket for now - -0.3.0 - 27-18-2018 --------------------------------------------------------------------------------- - -**Added** - -#. `#89 `_: Install pypi-hosted - mobans through requires syntax - -**Updated** - -#. `#96 `_: Fix for - FileNotFoundError for plugins -#. various documentation updates - -**Removed** - -#. `#88 `_: removed python 2.6 - support -#. removed python 3.3 support - -0.2.4 - 14-07-2018 --------------------------------------------------------------------------------- - -**Added** - -#. `#32 `_: option 1 copy a - directory without its subdirectories. -#. `#30 `_: command line template - option is ignore when a moban file is present - -0.2.3 - 10-07-2018 --------------------------------------------------------------------------------- - -**Added** - -#. `#76 `_: running moban as a - module from python command -#. `#32 `_: copy a directory - recusively -#. `#33 `_: template all files in - a directory - -0.2.2 - 16-06-2018 --------------------------------------------------------------------------------- - -**Added** - -#. `#31 `_: create directory if - missing during copying - -**Updated** - -#. `#28 `_: if a template has been - copied once before, it is skipped in the next moban call - -0.2.1 - 13-06-2018 --------------------------------------------------------------------------------- - -**Updated** - -#. templates using the same template engine will be templated as a group -#. update lml dependency to 0.0.3 - -0.2.0 - 11-06-2018 --------------------------------------------------------------------------------- - -**Added** - -#. `#18 `_: file exists test -#. `#23 `_: custom jinja plugins -#. `#26 `_: repr filter -#. `#47 `_: allow the expansion of - template engine -#. `#58 `_: allow template type - per template - -**Updated** - -#. `#34 `_: fix plural message if - single file is processed - -0.1.4 - 29-May-2018 --------------------------------------------------------------------------------- - -**Updated** - -#. `#21 `_: targets become - optional -#. `#19 `_: transfer symlink's - target file's file permission under unix/linux systems -#. `#16 `_: introduce copy key - word in mobanfile - -0.1.3 - 12-Mar-2018 --------------------------------------------------------------------------------- - -**Updated** - -#. handle unicode on python 2 - -0.1.2 - 10-Jan-2018 --------------------------------------------------------------------------------- - -**Added** - -#. `#13 `_: strip off new lines in - the templated file - -0.1.1 - 08-Jan-2018 --------------------------------------------------------------------------------- - -**Added** - -#. the ability to present a long text as multi-line paragraph with a custom - upper limit -#. speical filter expand github references: pull request and issues -#. `#15 `_: fix templating syntax - to enable python 2.6 - -0.1.0 - 19-Dec-2017 --------------------------------------------------------------------------------- - -**Added** - -#. `#14 `_, provide shell exit - code - -0.0.9 - 24-Nov-2017 --------------------------------------------------------------------------------- - -**Added** - -#. `#11 `_, recognize .moban.yaml - as well as .moban.yml. -#. `#9 `_, preserve file - permissions of the source template. -#. `-m` option is added to allow you to specify a custom moban file. kinda - related to issue 11. - -**Updated** - -#. use explicit version name: `moban_file_spec_version` so that `version` can be - used by users. `#10 `_ Please - note: moban_file_spec_version is reserved for future file spec upgrade. For - now, all files are assumed to be '1.0'. When there comes a new version i.e. - 2.0, new moban file based on 2.0 will have to include - 'moban_file_spec_version: 2.0' - -0.0.8 - 18-Nov-2017 --------------------------------------------------------------------------------- - -**Added** - -#. `#8 `_, verify the existence of - custom template and configuration directories. default .moban.td, .moban.cd - are ignored if they do not exist. - -**Updated** - -#. Colorize error messages and processing messages. crayons become a dependency. - -0.0.7 - 19-Jul-2017 --------------------------------------------------------------------------------- - -**Added** - -#. Bring the visibility of environment variable into jinja2 templating process: - `#7 `_ - -0.0.6 - 16-Jun-2017 --------------------------------------------------------------------------------- - -**Added** - -#. added '-f' flag to force moban to template all files despite of .moban.hashes - -**Updated** - -#. moban will not template target file in the situation where the changes - occured in target file than in the source: the template file + the data - configuration after moban has been applied. This new release will remove the - change during mobanization process. - -0.0.5 - 17-Mar-2017 --------------------------------------------------------------------------------- - -**Added** - -#. Create a default hash store when processing a moban file. It will save - unnecessary file write to the disc if the rendered content is not changed. -#. Added summary reports - -0.0.4 - 11-May-2016 --------------------------------------------------------------------------------- - -**Updated** - -#. Bug fix `#5 `_, should detect - duplicated targets in `.moban.yml` file. - -0.0.3 - 09-May-2016 --------------------------------------------------------------------------------- - -**Updated** - -#. Bug fix `#4 `_, keep trailing - new lines - -0.0.2 - 27-Apr-2016 --------------------------------------------------------------------------------- - -**Updated** - -#. Bug fix `#1 `_, failed to save - utf-8 characters - -0.0.1 - 23-Mar-2016 --------------------------------------------------------------------------------- - -**Added** - -#. Initial release +404: Not Found \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index 7f30573d..1becba2b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,85 +1 @@ -# -*- coding: utf-8 -*- -DESCRIPTION = ( - 'General purpose static text generator' + - '' -) -# Configuration file for the Sphinx documentation builder. -# -# This file only contains a selection of the most common options. For a full -# list see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html - -# -- Path setup -------------------------------------------------------------- - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) - -# -- Project information ----------------------------------------------------- - -project = 'moban' -copyright = '2017-2020 Onni Software Ltd.' -author = 'C. W.' -# The short X.Y version -version = '0.7.1' -# The full version, including alpha/beta/rc tags -release = '0.7.1' - -# -- General configuration --------------------------------------------------- - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.viewcode',] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = 'en' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This pattern also affects html_static_path and html_extra_path. -exclude_patterns = [] - - -# -- Options for HTML output ------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = 'alabaster' - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] - -# -- Extension configuration ------------------------------------------------- -# -- Options for intersphinx extension --------------------------------------- - -# Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'https://docs.python.org/3/': None} -# TODO: html_theme not configurable upstream -html_theme = 'default' - -# TODO: DESCRIPTION not configurable upstream -texinfo_documents = [ - ('index', 'moban', - 'moban Documentation', - 'Onni Software Ltd.', 'moban', - DESCRIPTION, - 'Miscellaneous'), -] -intersphinx_mapping.update({ -}) -master_doc = "index" +404: Not Found \ No newline at end of file diff --git a/min_requirements.txt b/min_requirements.txt index c3cac050..1becba2b 100644 --- a/min_requirements.txt +++ b/min_requirements.txt @@ -1,10 +1 @@ -ruamel.yaml==0.15.5;python_version == '3.4' -ruamel.yaml==0.15.42;python_version == '3.7' -ruamel.yaml==0.15.5;python_version != '3.4' and python_version < '3.7' -ruamel.yaml==0.15.98;python_version == '3.8' -jinja2==2.7.1 -lml==0.0.9 -appdirs==1.4.3 -crayons== 0.1.0 -fs==2.4.11 -jinja2-fsloader==0.2.0 +404: Not Found \ No newline at end of file diff --git a/moban/core/hashstore.py b/moban/core/hashstore.py index aae1d4b5..59b8b875 100644 --- a/moban/core/hashstore.py +++ b/moban/core/hashstore.py @@ -32,9 +32,14 @@ def is_file_changed(self, file_name, file_content, source_template): def _is_source_updated(self, file_name, file_content, source_template): changed = True - content = _mix( - file_content, oct(file_system.file_permissions(source_template)) - ) + content = file_content + try: + content = _mix( + file_content, + oct(file_system.file_permissions(source_template)), + ) + except: + pass content_hash = get_hash(content) if file_system.exists(file_name): if file_name in self.hashes: @@ -57,7 +62,10 @@ def save_hashes(self): def get_file_hash(afile): content = file_system.read_bytes(afile) - content = _mix(content, oct(file_system.file_permissions(afile))) + try: + content = _mix(content, oct(file_system.file_permissions(afile))) + except: + pass return get_hash(content) diff --git a/moban/core/moban_factory.py b/moban/core/moban_factory.py index a5081c7f..f7f90ba7 100644 --- a/moban/core/moban_factory.py +++ b/moban/core/moban_factory.py @@ -170,7 +170,7 @@ def apply_template(self, template_abs_path, template, data, output_file): self.buffered_writer.write_file_out( output_file, rendered_content ) - if not file_system.is_zip_alike_url(output_file): + if 1 == 2: file_system.file_permissions_copy( template_abs_path, output_file ) diff --git a/moban/core/utils.py b/moban/core/utils.py index 884e0e19..f01c6598 100644 --- a/moban/core/utils.py +++ b/moban/core/utils.py @@ -15,9 +15,9 @@ def verify_the_existence_of_directories(dirs): for directory in dirs: - if file_system.exists(directory): - results.append(directory) - continue + # if file_system.exists(directory): + # results.append(directory) + # continue should_I_ignore = ( constants.DEFAULT_CONFIGURATION_DIRNAME in directory or constants.DEFAULT_TEMPLATE_DIRNAME in directory @@ -26,7 +26,6 @@ def verify_the_existence_of_directories(dirs): # ignore pass else: - raise exceptions.DirectoryNotFound( - constants.MESSAGE_DIR_NOT_EXIST % directory - ) + results.append(directory) + return results diff --git a/mobanfile b/mobanfile index ecfc1d6a..1f887275 100644 --- a/mobanfile +++ b/mobanfile @@ -1,7 +1,6 @@ configuration: template_dir: - - "git://github.com/moremoban/pypi-mobans.git?submodule=true&brach=dev!/templates" - - "git://github.com/moremoban/pypi-mobans.git?submodule=true&brach=dev!/statics" + - "https://raw.githubusercontent.com/moremoban/pypi-mobans/dev/templates/" - ".moban.d" configuration: moban.yml targets: diff --git a/requirements.txt b/requirements.txt index 8ad69281..1becba2b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1 @@ -ruamel.yaml>=0.15.5,<=0.15.94;python_version == '3.4' -ruamel.yaml>=0.15.42;python_version == '3.7' -ruamel.yaml>=0.15.5;python_version != '3.4' and python_version < '3.7' -ruamel.yaml>=0.15.98;python_version == '3.8' -jinja2>=2.7.1 -lml>=0.0.9 -appdirs>=1.4.3 -crayons>= 0.1.0 -fs>=2.4.11 -jinja2-fsloader>=0.2.0 +404: Not Found \ No newline at end of file diff --git a/setup.py b/setup.py index 72af6c1f..1becba2b 100644 --- a/setup.py +++ b/setup.py @@ -1,273 +1 @@ -#!/usr/bin/env python3 - -""" -Template by pypi-mobans -""" - -import os -import sys -import codecs -import locale -import platform -from shutil import rmtree - -from setuptools import Command, setup, find_packages -from setuptools import __version__ as setuptools_version -from pkg_resources import parse_version - -import pkg_resources - -try: - import _markerlib.markers -except ImportError: - _markerlib = None - -PY2 = sys.version_info[0] == 2 -PY26 = PY2 and sys.version_info[1] < 7 -PY33 = sys.version_info < (3, 4) - -# Work around mbcs bug in distutils. -# http://bugs.python.org/issue10945 -# This work around is only if a project supports Python < 3.4 - -# Work around for locale not being set -try: - lc = locale.getlocale() - pf = platform.system() - if pf != "Windows" and lc == (None, None): - locale.setlocale(locale.LC_ALL, "C.UTF-8") -except (ValueError, UnicodeError, locale.Error): - locale.setlocale(locale.LC_ALL, "en_US.UTF-8") - -NAME = "moban" -AUTHOR = "C. W." -VERSION = "0.7.1" -EMAIL = "wangc_2011@hotmail.com" -LICENSE = "MIT" -ENTRY_POINTS = { - "console_scripts": [ - "moban = moban.main:main" - ], -} -DESCRIPTION = ( - "General purpose static text generator" -) -URL = "https://github.com/moremoban/moban" -DOWNLOAD_URL = "%s/archive/0.7.1.tar.gz" % URL -FILES = ["README.rst", "CONTRIBUTORS.rst", "CHANGELOG.rst"] -KEYWORDS = [ - "python", - "jinja2", - "moban", -] - -CLASSIFIERS = [ - "Topic :: Software Development :: Libraries", - "Programming Language :: Python", - "Intended Audience :: Developers", - "Programming Language :: Python :: 2.6", - "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3.3", - "Programming Language :: Python :: 3.4", - "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - -] - -PYTHON_REQUIRES = ">=3.6" - -INSTALL_REQUIRES = [ - "jinja2>=2.7.1", - "lml>=0.0.9", - "appdirs>=1.4.3", - "crayons>= 0.1.0", - "fs>=2.4.11", - "jinja2-fsloader>=0.2.0", -] -SETUP_COMMANDS = {} - -PACKAGES = find_packages(exclude=["ez_setup", "examples", "tests", "tests.*"]) -EXTRAS_REQUIRE = { - ":python_version == '3.4'": ["ruamel.yaml>=0.15.5,<=0.15.94"], - ":python_version == '3.7'": ["ruamel.yaml>=0.15.42"], - ":python_version != '3.4' and python_version < '3.7'": ["ruamel.yaml>=0.15.5"], - ":python_version == '3.8'": ["ruamel.yaml>=0.15.98"], -} -# You do not need to read beyond this line -PUBLISH_COMMAND = "{0} setup.py sdist bdist_wheel upload -r pypi".format(sys.executable) -GS_COMMAND = ("gs moban v0.7.1 " + - "Find 0.7.1 in changelog for more details") -NO_GS_MESSAGE = ("Automatic github release is disabled. " + - "Please install gease to enable it.") -UPLOAD_FAILED_MSG = ( - 'Upload failed. please run "%s" yourself.' % PUBLISH_COMMAND) -HERE = os.path.abspath(os.path.dirname(__file__)) - - -class PublishCommand(Command): - """Support setup.py upload.""" - - description = "Build and publish the package on github and pypi" - user_options = [] - - @staticmethod - def status(s): - """Prints things in bold.""" - print("\033[1m{0}\033[0m".format(s)) - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - try: - self.status("Removing previous builds...") - rmtree(os.path.join(HERE, "dist")) - rmtree(os.path.join(HERE, "build")) - rmtree(os.path.join(HERE, "moban.egg-info")) - except OSError: - pass - - self.status("Building Source and Wheel (universal) distribution...") - run_status = True - if has_gease(): - run_status = os.system(GS_COMMAND) == 0 - else: - self.status(NO_GS_MESSAGE) - if run_status: - if os.system(PUBLISH_COMMAND) != 0: - self.status(UPLOAD_FAILED_MSG) - - sys.exit() - - -SETUP_COMMANDS.update({ - "publish": PublishCommand -}) - - -def has_gease(): - """ - test if github release command is installed - - visit http://github.com/moremoban/gease for more info - """ - try: - import gease # noqa - return True - except ImportError: - return False - - -def read_files(*files): - """Read files into setup""" - text = "" - for single_file in files: - content = read(single_file) - text = text + content + "\n" - return text - - -def read(afile): - """Read a file into setup""" - the_relative_file = os.path.join(HERE, afile) - with codecs.open(the_relative_file, "r", "utf-8") as opened_file: - content = filter_out_test_code(opened_file) - content = "".join(list(content)) - return content - - -def filter_out_test_code(file_handle): - found_test_code = False - for line in file_handle.readlines(): - if line.startswith(".. testcode:"): - found_test_code = True - continue - if found_test_code is True: - if line.startswith(" "): - continue - else: - empty_line = line.strip() - if len(empty_line) == 0: - continue - else: - found_test_code = False - yield line - else: - for keyword in ["|version|", "|today|"]: - if keyword in line: - break - else: - yield line - - -# _markerlib.default_environment() obtains its data from _VARS -# and wraps it in another dict, but _markerlib_evaluate writes -# to the dict while it is iterating the keys, causing an error -# on Python 3 only. -# Replace _markerlib.default_environment to return a custom dict -# that has all the necessary markers, and ignores any writes. - -class Python3MarkerDict(dict): - - def __setitem__(self, key, value): - pass - - def pop(self, i=-1): - return self[i] - - -if _markerlib and sys.version_info[0] == 3: - env = _markerlib.markers._VARS - for key in list(env.keys()): - new_key = key.replace(".", "_") - if new_key != key: - env[new_key] = env[key] - - _markerlib.markers._VARS = Python3MarkerDict(env) - - def default_environment(): - return _markerlib.markers._VARS - - _markerlib.default_environment = default_environment - -# Avoid the very buggy pkg_resources.parser, which does not consistently -# recognise the markers needed by this setup.py -# See https://github.com/pypa/packaging/issues/72 for details -# Change this to setuptools 20.10.0 to support all markers. -if pkg_resources: - if parse_version(setuptools_version) < parse_version("18.5"): - MarkerEvaluation = pkg_resources.MarkerEvaluation - - del pkg_resources.parser - pkg_resources.evaluate_marker = MarkerEvaluation._markerlib_evaluate - MarkerEvaluation.evaluate_marker = MarkerEvaluation._markerlib_evaluate - -if __name__ == "__main__": - setup( - test_suite="tests", - name=NAME, - author=AUTHOR, - version=VERSION, - author_email=EMAIL, - description=DESCRIPTION, - url=URL, - download_url=DOWNLOAD_URL, - long_description=read_files(*FILES), - license=LICENSE, - keywords=KEYWORDS, - python_requires=PYTHON_REQUIRES, - extras_require=EXTRAS_REQUIRE, - tests_require=["nose"], - install_requires=INSTALL_REQUIRES, - packages=PACKAGES, - include_package_data=True, - zip_safe=False, - entry_points=ENTRY_POINTS, - classifiers=CLASSIFIERS, - cmdclass=SETUP_COMMANDS - ) +404: Not Found \ No newline at end of file From eca76cde090db2e1f96f6851fedcee3a3545d795 Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 1 May 2020 00:08:44 +0100 Subject: [PATCH 02/14] :tractor: revert the changes in static files --- .github/workflows/pythonpublish.yml | 27 +- .travis.yml | 52 ++- CHANGELOG.rst | 583 +++++++++++++++++++++++++++- docs/conf.py | 86 +++- min_requirements.txt | 11 +- mobanfile | 3 +- requirements.txt | 11 +- setup.py | 274 ++++++++++++- 8 files changed, 1039 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml index 1becba2b..9e7ec429 100644 --- a/.github/workflows/pythonpublish.yml +++ b/.github/workflows/pythonpublish.yml @@ -1 +1,26 @@ -404: Not Found \ No newline at end of file +name: Upload Python Package + +on: + release: + types: [created] + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + - name: Build and publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python setup.py sdist bdist_wheel + twine upload dist/* diff --git a/.travis.yml b/.travis.yml index 1becba2b..ad53eb84 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1 +1,51 @@ -404: Not Found \ No newline at end of file +sudo: false +dist: xenial +language: python +notifications: + email: false +python: + - 3.7 + - 3.6 + - 3.8 +env: + - MINREQ=0 + - MINREQ=1 + +stages: + - lint + - moban + - test + + +.lint: &lint + git: + submodules: false + python: 3.6 + stage: lint + script: make install_test format git-diff-check lint + +.moban: &moban + python: 3.6 + stage: moban + install: pip install moban>=0.0.4 gitfs2 pypifs + script: make update git-diff-check + +jobs: + include: + - *moban + - *lint + +stage: test + +before_install: + - if [[ -f min_requirements.txt && "$MINREQ" -eq 1 ]]; then + mv min_requirements.txt requirements.txt ; + fi + - test ! -f rnd_requirements.txt || + pip install --no-deps -r rnd_requirements.txt + - test ! -f rnd_requirements.txt || pip install -r rnd_requirements.txt ; + - pip install -r tests/requirements.txt +script: + - make test +after_success: + codecov diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1becba2b..8f45a05a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1 +1,582 @@ -404: Not Found \ No newline at end of file +Change log +================================================================================ + +0.7.1 - 25.04.2020 +-------------------------------------------------------------------------------- + +**Fixed** + +#. `#365 `_: regression was + introduced by v0.6.5. If you uses mobanfile as data configuration file, you + are very likely to have this show stopper. Please upgrade to this version. + +0.7.0 - 18.01.2020 +-------------------------------------------------------------------------------- + +**Removed** + +#. `#360 `_: make gitfs2 and + pypifs optional. +#. `#303 `_: python 2.7 support + is dropped. + +**Updated** + +#. `#360 `_: show friendlier + error when unknown protocol exception was raised. + +0.6.8 - 7.12.2019 +-------------------------------------------------------------------------------- + +**Updated** + +#. since version 0.5.0, when rendering a single file or string, moban would + report 'Templated 1 of 0 files', which should have been 'Templated 1 file.' + +**Removed** + +#. python 3.4 support is gone because colorama requires Python '>=2.7, !=3.0.*, + !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*' + +0.6.7 - 1.12.2019 +-------------------------------------------------------------------------------- + +**Updated** + +#. no verbose for error, -v for warning, -vv for warning+info, -vvv for + warning+info+debug +#. `#351 `_, show template plugin + name, i.e. 'copying' for copy instead of 'templating' + +**Removed** + +#. Message: 'Warning: Attempting to use environment vars as data...' became + warning log +#. Message: 'Warning: Both data.yml and /.../.moban.cd/data.yml does not exist' + became warning log +#. with -v, you would see them in such a situation + +0.6.6 - 10.11.2019 +-------------------------------------------------------------------------------- + +**Added** + +#. support moban file inheritance. one base moban file and child repos can + inherit and override + +0.6.5 - 13.10.2019 +-------------------------------------------------------------------------------- + +**Added** + +#. `#335 `_: support intermediate + targets in moban file + +0.6.4 - 4.10.2019 +-------------------------------------------------------------------------------- + +**Updated** + +#. Command options have been grouped. --template_type became --template-type +#. Increment gitfs2 to version 0.0.2. `gitfs#4 + `_ + +0.6.3 - 25.09.2019 +-------------------------------------------------------------------------------- + +**Added** + +#. `#260 `_: jinja-cli parity: + support command line pipe stream. + +0.6.2 - 15.09.2019 +-------------------------------------------------------------------------------- + +**Added** + +#. `#322 `_: Implicit targets + with template extensions default to copy +#. `#257 `_: '-e' to load + extensions for template engines, i.e. jinja2 +#. `#333 `_: command line + template fails with version 0.6.1 + +0.6.1 - 10.09.2019 +-------------------------------------------------------------------------------- + +**Fixed** + +#. `#328 `_: update backward + compatibility + +0.6.0 - 10.09.2019 +-------------------------------------------------------------------------------- + +**Added** + +#. `#205 `_: support + `pyFilesystem2 `_ +#. `#185 `_: -v will enable moban + application logging for development. And -V is for version. +#. `#325 `_: -vv show debug trace +#. `#126 `_: Allow mobanfile to + include data from arbitrary config files +#. `#256 `_: jinja2-cli parity: + '-d hello=world' to define custom variable on cli + +**Updated** + +#. `#275 `_: fix moban 0.4.5 test + failures on openSUSE Tumbleweed + +0.5.0 - 14.07.2019 +-------------------------------------------------------------------------------- + +**Updated** + +#. `#277 `_: Restored dependency + git-url-parse, replacing incompatible giturlparse which was used during moban + 0.4.x +#. `#281 `_: Fixed unicode + support on Python 2.7 +#. `#274 `_: Updated ruamel.yaml + dependency pins to restore support for Python 3.4, and prevent installation + of versions that can not be installed on Python 3.7 +#. `#285 `_: Fixed CI testing of + minimum requirements +#. `#271 `_: Fixed repository + caching bug preventing branch switching +#. `#292 `_: Reformatted YAML + files according to yamllint rules +#. `#291 `_: Fixed filename typos + in README +#. `#280 `_: Added CI to ensure + repository is in sync with upstream +#. `#280 `_: sync setup.py from + pypi-mobans + +0.4.5 - 07.07.2019 +-------------------------------------------------------------------------------- + +**Updated** + +#. `#271 `_: support git branch + change in later run. + +0.4.4 - 26.05.2019 +-------------------------------------------------------------------------------- + +**Updated** + +#. `#265 `_: Use simple `read + binary` to read instead of encoding + +0.4.3 - 16.03.2019 +-------------------------------------------------------------------------------- + +**Removed** + +#. `#253 `_: symbolic link in + regression pack causes python setup.py to do recursive include + +**Added** + +#. `#209 `_: Alert moban user + when `git` is not available and is used. + +**Updated** + +#. `#261 `_: since moban group + template files per template type, this fill use first come first register to + order moban group + +0.4.2 - 08.03.2019 +-------------------------------------------------------------------------------- + +**Added** + +#. `#234 `_: Define template + parameters on the fly inside `targets` section +#. `#62 `_: select a group target + to run + +**Updated** + +#. `#180 `_: No longer two + statistics will be shown in v0.4.x. legacy copy targets are injected into a + normal targets. cli target is made a clear priority. +#. `#244 `_: version 0.4.2 is + first version which would work perfectly on windows since 17 Nov 2018. Note + that: file permissions are not used on windows. Why the date? because + samefile is not avaiable on windows, causing unit tests to fail hence it lead + to my conclusion that moban version between 17 Nov 2018 and March 2019 wont + work well on Windows. + +0.4.1 - 28.02.2019 +-------------------------------------------------------------------------------- + +**Added** + +#. `#235 `_: user defined + template types so that custom file extensions, template configurations can be + controlled by moban user +#. `#232 `_: the package + dependencies have been fine tuning to lower versions, most of them are dated + back to 2017. + +0.4.0 - 20.02.2019 +-------------------------------------------------------------------------------- + +**Added** + +#. `#165 `_: Copy as plugins + +**Updated** + +#. `#219 `_: git clone depth set + to 2 +#. `#186 `_: lowest dependecy on + ruamel.yaml is 0.15.5, Jun 2017 + +0.3.10 - 03.02.2019 +-------------------------------------------------------------------------------- + +**Added** + +#. `#174 `_: Store git cache in + XDG_CACHE_DIR +#. `#107 `_: Add -v to show + current moban version +#. `#164 `_: support additional + data formats + +**Updated** + +#. `#178 `_: UnboundLocalError: + local variable 'target' referenced before assignment +#. `#169 `_: uses GitPython + instead of barebone git commands + +0.3.9 - 18-1-2019 +-------------------------------------------------------------------------------- + +**Updated** + +#. `#90 `_: allow adding extra + jinja2 extensions. `jinja2.ext.do`, `jinja2.ext.loopcontrols` are included by + default. what's more, any other template enigne are eligible for extension + additions. +#. `#158 `_: Empty file + base_engine.py is finally removed + +0.3.8 - 12-1-2019 +-------------------------------------------------------------------------------- + +**Updated** + +#. `#141 `_: disable file + permissions copy feature and not to check file permission changes on windows. +#. `#154 `_: introduce first ever + positional argument for string base template. +#. `#157 `_: the exit code + behavior changed. for backward compactibility please use --exit-code. + Otherwise, moban will not tell if there is any changes. + +0.3.7 - 6-1-2019 +-------------------------------------------------------------------------------- + +**Updated** + +#. `#146 `_: added a low-setup + usage mode via environment variables to moban +#. `#148 `_: include test related + files in the package for package validation when distributing via linux + system, i.e. OpenSuse + +0.3.6 - 30-12-2018 +-------------------------------------------------------------------------------- + +**Updated** + +#. `#143 `_: moban shall report + permission error and continue the rest of the copying task. +#. `#122 `_: Since 0.3.6, moban + is tested on windows and macos too, using azure build pipelines. It is + already tested extensively on travis-ci on linux os. + +0.3.5 - 10-12-2018 +-------------------------------------------------------------------------------- + +**Updated** + +#. `#37 `_: moban will report line + number where the value is empty and the name of mobanfile. Switch from pyyaml + to ruamel.yaml. + +0.3.4.1 - 28-11-2018 +-------------------------------------------------------------------------------- + +**Updated** + +#. `#137 `_: missing + contributors.rst file + +0.3.4 - 18-11-2018 +-------------------------------------------------------------------------------- + +**Added** + +#. global variables to store the target and template file names in the jinja2 + engine +#. moban-handlebars is tested to work well with this version and above + +**Updated** + +#. Template engine interface has been clarified and documented + +0.3.3 - 05-11-2018 +-------------------------------------------------------------------------------- + +**Added** + +#. alternative and expanded syntax for requires, so as to accomendate github + submodule recursive + +0.3.2 - 04-11-2018 +-------------------------------------------------------------------------------- + +**Added** + +#. configuration dirs may be located by `requires`, i.e. configuration files may + be in a python package or git repository. + +0.3.1 - 02-11-2018 +-------------------------------------------------------------------------------- + +**Added** + +#. `#97 `_: requires will clone a + repo if given. Note: only github, gitlab, bitbucket for now + +0.3.0 - 27-18-2018 +-------------------------------------------------------------------------------- + +**Added** + +#. `#89 `_: Install pypi-hosted + mobans through requires syntax + +**Updated** + +#. `#96 `_: Fix for + FileNotFoundError for plugins +#. various documentation updates + +**Removed** + +#. `#88 `_: removed python 2.6 + support +#. removed python 3.3 support + +0.2.4 - 14-07-2018 +-------------------------------------------------------------------------------- + +**Added** + +#. `#32 `_: option 1 copy a + directory without its subdirectories. +#. `#30 `_: command line template + option is ignore when a moban file is present + +0.2.3 - 10-07-2018 +-------------------------------------------------------------------------------- + +**Added** + +#. `#76 `_: running moban as a + module from python command +#. `#32 `_: copy a directory + recusively +#. `#33 `_: template all files in + a directory + +0.2.2 - 16-06-2018 +-------------------------------------------------------------------------------- + +**Added** + +#. `#31 `_: create directory if + missing during copying + +**Updated** + +#. `#28 `_: if a template has been + copied once before, it is skipped in the next moban call + +0.2.1 - 13-06-2018 +-------------------------------------------------------------------------------- + +**Updated** + +#. templates using the same template engine will be templated as a group +#. update lml dependency to 0.0.3 + +0.2.0 - 11-06-2018 +-------------------------------------------------------------------------------- + +**Added** + +#. `#18 `_: file exists test +#. `#23 `_: custom jinja plugins +#. `#26 `_: repr filter +#. `#47 `_: allow the expansion of + template engine +#. `#58 `_: allow template type + per template + +**Updated** + +#. `#34 `_: fix plural message if + single file is processed + +0.1.4 - 29-May-2018 +-------------------------------------------------------------------------------- + +**Updated** + +#. `#21 `_: targets become + optional +#. `#19 `_: transfer symlink's + target file's file permission under unix/linux systems +#. `#16 `_: introduce copy key + word in mobanfile + +0.1.3 - 12-Mar-2018 +-------------------------------------------------------------------------------- + +**Updated** + +#. handle unicode on python 2 + +0.1.2 - 10-Jan-2018 +-------------------------------------------------------------------------------- + +**Added** + +#. `#13 `_: strip off new lines in + the templated file + +0.1.1 - 08-Jan-2018 +-------------------------------------------------------------------------------- + +**Added** + +#. the ability to present a long text as multi-line paragraph with a custom + upper limit +#. speical filter expand github references: pull request and issues +#. `#15 `_: fix templating syntax + to enable python 2.6 + +0.1.0 - 19-Dec-2017 +-------------------------------------------------------------------------------- + +**Added** + +#. `#14 `_, provide shell exit + code + +0.0.9 - 24-Nov-2017 +-------------------------------------------------------------------------------- + +**Added** + +#. `#11 `_, recognize .moban.yaml + as well as .moban.yml. +#. `#9 `_, preserve file + permissions of the source template. +#. `-m` option is added to allow you to specify a custom moban file. kinda + related to issue 11. + +**Updated** + +#. use explicit version name: `moban_file_spec_version` so that `version` can be + used by users. `#10 `_ Please + note: moban_file_spec_version is reserved for future file spec upgrade. For + now, all files are assumed to be '1.0'. When there comes a new version i.e. + 2.0, new moban file based on 2.0 will have to include + 'moban_file_spec_version: 2.0' + +0.0.8 - 18-Nov-2017 +-------------------------------------------------------------------------------- + +**Added** + +#. `#8 `_, verify the existence of + custom template and configuration directories. default .moban.td, .moban.cd + are ignored if they do not exist. + +**Updated** + +#. Colorize error messages and processing messages. crayons become a dependency. + +0.0.7 - 19-Jul-2017 +-------------------------------------------------------------------------------- + +**Added** + +#. Bring the visibility of environment variable into jinja2 templating process: + `#7 `_ + +0.0.6 - 16-Jun-2017 +-------------------------------------------------------------------------------- + +**Added** + +#. added '-f' flag to force moban to template all files despite of .moban.hashes + +**Updated** + +#. moban will not template target file in the situation where the changes + occured in target file than in the source: the template file + the data + configuration after moban has been applied. This new release will remove the + change during mobanization process. + +0.0.5 - 17-Mar-2017 +-------------------------------------------------------------------------------- + +**Added** + +#. Create a default hash store when processing a moban file. It will save + unnecessary file write to the disc if the rendered content is not changed. +#. Added summary reports + +0.0.4 - 11-May-2016 +-------------------------------------------------------------------------------- + +**Updated** + +#. Bug fix `#5 `_, should detect + duplicated targets in `.moban.yml` file. + +0.0.3 - 09-May-2016 +-------------------------------------------------------------------------------- + +**Updated** + +#. Bug fix `#4 `_, keep trailing + new lines + +0.0.2 - 27-Apr-2016 +-------------------------------------------------------------------------------- + +**Updated** + +#. Bug fix `#1 `_, failed to save + utf-8 characters + +0.0.1 - 23-Mar-2016 +-------------------------------------------------------------------------------- + +**Added** + +#. Initial release diff --git a/docs/conf.py b/docs/conf.py index 1becba2b..7f30573d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1 +1,85 @@ -404: Not Found \ No newline at end of file +# -*- coding: utf-8 -*- +DESCRIPTION = ( + 'General purpose static text generator' + + '' +) +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + +# -- Project information ----------------------------------------------------- + +project = 'moban' +copyright = '2017-2020 Onni Software Ltd.' +author = 'C. W.' +# The short X.Y version +version = '0.7.1' +# The full version, including alpha/beta/rc tags +release = '0.7.1' + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx', 'sphinx.ext.viewcode',] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = 'en' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = [] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# -- Extension configuration ------------------------------------------------- +# -- Options for intersphinx extension --------------------------------------- + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = {'https://docs.python.org/3/': None} +# TODO: html_theme not configurable upstream +html_theme = 'default' + +# TODO: DESCRIPTION not configurable upstream +texinfo_documents = [ + ('index', 'moban', + 'moban Documentation', + 'Onni Software Ltd.', 'moban', + DESCRIPTION, + 'Miscellaneous'), +] +intersphinx_mapping.update({ +}) +master_doc = "index" diff --git a/min_requirements.txt b/min_requirements.txt index 1becba2b..c3cac050 100644 --- a/min_requirements.txt +++ b/min_requirements.txt @@ -1 +1,10 @@ -404: Not Found \ No newline at end of file +ruamel.yaml==0.15.5;python_version == '3.4' +ruamel.yaml==0.15.42;python_version == '3.7' +ruamel.yaml==0.15.5;python_version != '3.4' and python_version < '3.7' +ruamel.yaml==0.15.98;python_version == '3.8' +jinja2==2.7.1 +lml==0.0.9 +appdirs==1.4.3 +crayons== 0.1.0 +fs==2.4.11 +jinja2-fsloader==0.2.0 diff --git a/mobanfile b/mobanfile index 1f887275..ecfc1d6a 100644 --- a/mobanfile +++ b/mobanfile @@ -1,6 +1,7 @@ configuration: template_dir: - - "https://raw.githubusercontent.com/moremoban/pypi-mobans/dev/templates/" + - "git://github.com/moremoban/pypi-mobans.git?submodule=true&brach=dev!/templates" + - "git://github.com/moremoban/pypi-mobans.git?submodule=true&brach=dev!/statics" - ".moban.d" configuration: moban.yml targets: diff --git a/requirements.txt b/requirements.txt index 1becba2b..8ad69281 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,10 @@ -404: Not Found \ No newline at end of file +ruamel.yaml>=0.15.5,<=0.15.94;python_version == '3.4' +ruamel.yaml>=0.15.42;python_version == '3.7' +ruamel.yaml>=0.15.5;python_version != '3.4' and python_version < '3.7' +ruamel.yaml>=0.15.98;python_version == '3.8' +jinja2>=2.7.1 +lml>=0.0.9 +appdirs>=1.4.3 +crayons>= 0.1.0 +fs>=2.4.11 +jinja2-fsloader>=0.2.0 diff --git a/setup.py b/setup.py index 1becba2b..72af6c1f 100644 --- a/setup.py +++ b/setup.py @@ -1 +1,273 @@ -404: Not Found \ No newline at end of file +#!/usr/bin/env python3 + +""" +Template by pypi-mobans +""" + +import os +import sys +import codecs +import locale +import platform +from shutil import rmtree + +from setuptools import Command, setup, find_packages +from setuptools import __version__ as setuptools_version +from pkg_resources import parse_version + +import pkg_resources + +try: + import _markerlib.markers +except ImportError: + _markerlib = None + +PY2 = sys.version_info[0] == 2 +PY26 = PY2 and sys.version_info[1] < 7 +PY33 = sys.version_info < (3, 4) + +# Work around mbcs bug in distutils. +# http://bugs.python.org/issue10945 +# This work around is only if a project supports Python < 3.4 + +# Work around for locale not being set +try: + lc = locale.getlocale() + pf = platform.system() + if pf != "Windows" and lc == (None, None): + locale.setlocale(locale.LC_ALL, "C.UTF-8") +except (ValueError, UnicodeError, locale.Error): + locale.setlocale(locale.LC_ALL, "en_US.UTF-8") + +NAME = "moban" +AUTHOR = "C. W." +VERSION = "0.7.1" +EMAIL = "wangc_2011@hotmail.com" +LICENSE = "MIT" +ENTRY_POINTS = { + "console_scripts": [ + "moban = moban.main:main" + ], +} +DESCRIPTION = ( + "General purpose static text generator" +) +URL = "https://github.com/moremoban/moban" +DOWNLOAD_URL = "%s/archive/0.7.1.tar.gz" % URL +FILES = ["README.rst", "CONTRIBUTORS.rst", "CHANGELOG.rst"] +KEYWORDS = [ + "python", + "jinja2", + "moban", +] + +CLASSIFIERS = [ + "Topic :: Software Development :: Libraries", + "Programming Language :: Python", + "Intended Audience :: Developers", + "Programming Language :: Python :: 2.6", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + +] + +PYTHON_REQUIRES = ">=3.6" + +INSTALL_REQUIRES = [ + "jinja2>=2.7.1", + "lml>=0.0.9", + "appdirs>=1.4.3", + "crayons>= 0.1.0", + "fs>=2.4.11", + "jinja2-fsloader>=0.2.0", +] +SETUP_COMMANDS = {} + +PACKAGES = find_packages(exclude=["ez_setup", "examples", "tests", "tests.*"]) +EXTRAS_REQUIRE = { + ":python_version == '3.4'": ["ruamel.yaml>=0.15.5,<=0.15.94"], + ":python_version == '3.7'": ["ruamel.yaml>=0.15.42"], + ":python_version != '3.4' and python_version < '3.7'": ["ruamel.yaml>=0.15.5"], + ":python_version == '3.8'": ["ruamel.yaml>=0.15.98"], +} +# You do not need to read beyond this line +PUBLISH_COMMAND = "{0} setup.py sdist bdist_wheel upload -r pypi".format(sys.executable) +GS_COMMAND = ("gs moban v0.7.1 " + + "Find 0.7.1 in changelog for more details") +NO_GS_MESSAGE = ("Automatic github release is disabled. " + + "Please install gease to enable it.") +UPLOAD_FAILED_MSG = ( + 'Upload failed. please run "%s" yourself.' % PUBLISH_COMMAND) +HERE = os.path.abspath(os.path.dirname(__file__)) + + +class PublishCommand(Command): + """Support setup.py upload.""" + + description = "Build and publish the package on github and pypi" + user_options = [] + + @staticmethod + def status(s): + """Prints things in bold.""" + print("\033[1m{0}\033[0m".format(s)) + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + try: + self.status("Removing previous builds...") + rmtree(os.path.join(HERE, "dist")) + rmtree(os.path.join(HERE, "build")) + rmtree(os.path.join(HERE, "moban.egg-info")) + except OSError: + pass + + self.status("Building Source and Wheel (universal) distribution...") + run_status = True + if has_gease(): + run_status = os.system(GS_COMMAND) == 0 + else: + self.status(NO_GS_MESSAGE) + if run_status: + if os.system(PUBLISH_COMMAND) != 0: + self.status(UPLOAD_FAILED_MSG) + + sys.exit() + + +SETUP_COMMANDS.update({ + "publish": PublishCommand +}) + + +def has_gease(): + """ + test if github release command is installed + + visit http://github.com/moremoban/gease for more info + """ + try: + import gease # noqa + return True + except ImportError: + return False + + +def read_files(*files): + """Read files into setup""" + text = "" + for single_file in files: + content = read(single_file) + text = text + content + "\n" + return text + + +def read(afile): + """Read a file into setup""" + the_relative_file = os.path.join(HERE, afile) + with codecs.open(the_relative_file, "r", "utf-8") as opened_file: + content = filter_out_test_code(opened_file) + content = "".join(list(content)) + return content + + +def filter_out_test_code(file_handle): + found_test_code = False + for line in file_handle.readlines(): + if line.startswith(".. testcode:"): + found_test_code = True + continue + if found_test_code is True: + if line.startswith(" "): + continue + else: + empty_line = line.strip() + if len(empty_line) == 0: + continue + else: + found_test_code = False + yield line + else: + for keyword in ["|version|", "|today|"]: + if keyword in line: + break + else: + yield line + + +# _markerlib.default_environment() obtains its data from _VARS +# and wraps it in another dict, but _markerlib_evaluate writes +# to the dict while it is iterating the keys, causing an error +# on Python 3 only. +# Replace _markerlib.default_environment to return a custom dict +# that has all the necessary markers, and ignores any writes. + +class Python3MarkerDict(dict): + + def __setitem__(self, key, value): + pass + + def pop(self, i=-1): + return self[i] + + +if _markerlib and sys.version_info[0] == 3: + env = _markerlib.markers._VARS + for key in list(env.keys()): + new_key = key.replace(".", "_") + if new_key != key: + env[new_key] = env[key] + + _markerlib.markers._VARS = Python3MarkerDict(env) + + def default_environment(): + return _markerlib.markers._VARS + + _markerlib.default_environment = default_environment + +# Avoid the very buggy pkg_resources.parser, which does not consistently +# recognise the markers needed by this setup.py +# See https://github.com/pypa/packaging/issues/72 for details +# Change this to setuptools 20.10.0 to support all markers. +if pkg_resources: + if parse_version(setuptools_version) < parse_version("18.5"): + MarkerEvaluation = pkg_resources.MarkerEvaluation + + del pkg_resources.parser + pkg_resources.evaluate_marker = MarkerEvaluation._markerlib_evaluate + MarkerEvaluation.evaluate_marker = MarkerEvaluation._markerlib_evaluate + +if __name__ == "__main__": + setup( + test_suite="tests", + name=NAME, + author=AUTHOR, + version=VERSION, + author_email=EMAIL, + description=DESCRIPTION, + url=URL, + download_url=DOWNLOAD_URL, + long_description=read_files(*FILES), + license=LICENSE, + keywords=KEYWORDS, + python_requires=PYTHON_REQUIRES, + extras_require=EXTRAS_REQUIRE, + tests_require=["nose"], + install_requires=INSTALL_REQUIRES, + packages=PACKAGES, + include_package_data=True, + zip_safe=False, + entry_points=ENTRY_POINTS, + classifiers=CLASSIFIERS, + cmdclass=SETUP_COMMANDS + ) From 862b789a4b1923b8d35743c6f0f2d59a60ca64f5 Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 1 May 2020 18:23:21 +0100 Subject: [PATCH 03/14] :lipstick: make the implementation proper --- moban/core/hashstore.py | 7 +++++-- moban/core/moban_factory.py | 14 +++++++++----- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/moban/core/hashstore.py b/moban/core/hashstore.py index 59b8b875..1b660b61 100644 --- a/moban/core/hashstore.py +++ b/moban/core/hashstore.py @@ -1,6 +1,7 @@ import json import hashlib +from fs.errors import NoSysPath from moban import constants from moban.externals import file_system @@ -38,7 +39,8 @@ def _is_source_updated(self, file_name, file_content, source_template): file_content, oct(file_system.file_permissions(source_template)), ) - except: + except NoSysPath: + # HttpFs does not have getsyspath pass content_hash = get_hash(content) if file_system.exists(file_name): @@ -64,7 +66,8 @@ def get_file_hash(afile): content = file_system.read_bytes(afile) try: content = _mix(content, oct(file_system.file_permissions(afile))) - except: + except NoSysPath: + # HttpFs does not have getsyspath pass return get_hash(content) diff --git a/moban/core/moban_factory.py b/moban/core/moban_factory.py index f7f90ba7..c8d15d8a 100644 --- a/moban/core/moban_factory.py +++ b/moban/core/moban_factory.py @@ -3,7 +3,7 @@ import logging from collections import defaultdict -from fs.errors import ResourceNotFound +from fs.errors import ResourceNotFound, NoSysPath from lml.plugin import PluginManager from moban import constants, exceptions @@ -170,10 +170,14 @@ def apply_template(self, template_abs_path, template, data, output_file): self.buffered_writer.write_file_out( output_file, rendered_content ) - if 1 == 2: - file_system.file_permissions_copy( - template_abs_path, output_file - ) + if not file_system.is_zip_alike_url(output_file): + try: + file_system.file_permissions_copy( + template_abs_path, output_file + ) + except NoSysPath: + # HttpFs does not have getsyspath + pass return flag except exceptions.FileNotFound: # the template is a string from command line From d2f01287480bed5d44fdfafbf7fd874cb2154fce Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 1 May 2020 18:27:07 +0100 Subject: [PATCH 04/14] :hammer: code refactoring --- moban/core/hashstore.py | 3 ++- moban/core/moban_factory.py | 2 +- moban/core/utils.py | 24 ++++++------------------ 3 files changed, 9 insertions(+), 20 deletions(-) diff --git a/moban/core/hashstore.py b/moban/core/hashstore.py index 1b660b61..0f3ddfbb 100644 --- a/moban/core/hashstore.py +++ b/moban/core/hashstore.py @@ -2,6 +2,7 @@ import hashlib from fs.errors import NoSysPath + from moban import constants from moban.externals import file_system @@ -40,7 +41,7 @@ def _is_source_updated(self, file_name, file_content, source_template): oct(file_system.file_permissions(source_template)), ) except NoSysPath: - # HttpFs does not have getsyspath + # HttpFs does not have getsyspath pass content_hash = get_hash(content) if file_system.exists(file_name): diff --git a/moban/core/moban_factory.py b/moban/core/moban_factory.py index c8d15d8a..1592ea4f 100644 --- a/moban/core/moban_factory.py +++ b/moban/core/moban_factory.py @@ -3,7 +3,7 @@ import logging from collections import defaultdict -from fs.errors import ResourceNotFound, NoSysPath +from fs.errors import NoSysPath, ResourceNotFound from lml.plugin import PluginManager from moban import constants, exceptions diff --git a/moban/core/utils.py b/moban/core/utils.py index f01c6598..5f55f3c1 100644 --- a/moban/core/utils.py +++ b/moban/core/utils.py @@ -1,31 +1,19 @@ import logging -from moban import constants, exceptions -from moban.externals import file_system +from moban import constants LOG = logging.getLogger(__name__) def verify_the_existence_of_directories(dirs): - LOG.debug("Verifying the existence: %s", dirs) if not isinstance(dirs, list): dirs = [dirs] - results = [] - - for directory in dirs: - - # if file_system.exists(directory): - # results.append(directory) - # continue - should_I_ignore = ( + return [ + directory + for directory in dirs + if not ( constants.DEFAULT_CONFIGURATION_DIRNAME in directory or constants.DEFAULT_TEMPLATE_DIRNAME in directory ) - if should_I_ignore: - # ignore - pass - else: - results.append(directory) - - return results + ] From 4f8f18ba699589483f16c98192e1f31e887ca31f Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 1 May 2020 19:30:27 +0100 Subject: [PATCH 05/14] :green_heart: fix code to pass unit tests --- moban/core/utils.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/moban/core/utils.py b/moban/core/utils.py index 5f55f3c1..f00a89fd 100644 --- a/moban/core/utils.py +++ b/moban/core/utils.py @@ -1,6 +1,7 @@ import logging from moban import constants +from moban.externals import file_system LOG = logging.getLogger(__name__) @@ -9,7 +10,7 @@ def verify_the_existence_of_directories(dirs): if not isinstance(dirs, list): dirs = [dirs] - return [ + dirs = [ directory for directory in dirs if not ( @@ -17,3 +18,9 @@ def verify_the_existence_of_directories(dirs): or constants.DEFAULT_TEMPLATE_DIRNAME in directory ) ] + if file_system.exists(constants.DEFAULT_CONFIGURATION_DIRNAME): + dirs.append(constants.DEFAULT_CONFIGURATION_DIRNAME) + + if file_system.exists(constants.DEFAULT_TEMPLATE_DIRNAME): + dirs.append(constants.DEFAULT_TEMPLATE_DIRNAME) + return dirs From 79cdf2878b671c23dba43d843b16ad46739e1ddf Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 1 May 2020 20:10:47 +0100 Subject: [PATCH 06/14] :books: update documentation --- .moban.cd/changelog.yml | 6 ++++++ CHANGELOG.rst | 8 ++++++++ README.rst | 17 +++++++++++++++-- 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/.moban.cd/changelog.yml b/.moban.cd/changelog.yml index db4b4c7b..df66c6a8 100644 --- a/.moban.cd/changelog.yml +++ b/.moban.cd/changelog.yml @@ -1,6 +1,12 @@ name: moban organisation: moremoban releases: + - changes: + - action: Added + details: + - "Partial support for templates and configuration files over HTTP(S) protocol with httpfs." + date: tbd + version: 0.7.2 - changes: - action: Fixed details: diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8f45a05a..b8663327 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,14 @@ Change log ================================================================================ +0.7.2 - tbd +-------------------------------------------------------------------------------- + +**Added** + +#. Partial support for templates and configuration files over HTTP(S) protocol + with httpfs. + 0.7.1 - 25.04.2020 -------------------------------------------------------------------------------- diff --git a/README.rst b/README.rst index 93392b80..ca15aa7c 100644 --- a/README.rst +++ b/README.rst @@ -56,6 +56,7 @@ organisation. And here is a list of other usages: +1. `Django Mobans `_, templates for django, docker etc. 1. `Math Sheets `_, generate custom math sheets in pdf @@ -160,7 +161,19 @@ Moban in live action: All use cases are documented `here `_ -Work with files in a git repo +Templates and configuration files over HTTP(S) +================================================================================ + +`httpfs `_ should be installed first. + +.. code-block:: bash + + $ moban -t 'https://raw.githubusercontent.com/moremoban/pypi-mobans/dev/templates/_version.py.jj2'\ + -c 'https://raw.githubusercontent.com/moremoban/pypi-mobans/dev/config/data.yml'\ + -o _version.py + + +Templates and configuration files in a git repo ================================================================================ `gitfs2 `_ is optional since v0.7.0 but was @@ -182,7 +195,7 @@ You can do the following with moban: __author__ = "C.W." -Work with files in a python package +Templates and configuration files in a python package ================================================================================ `pypifs `_ is optional since v0.7.0 but From 9eeb7da8dd77ebe637f0774ab81d8d3c3145b27b Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 1 May 2020 21:57:16 +0100 Subject: [PATCH 07/14] :microscope: :books: regression test and documentation --- docs/README.rst | 3 +- docs/index.rst | 1 + docs/level-24-files-over-http/.moban.yml | 9 ++++++ docs/level-24-files-over-http/README.rst | 39 ++++++++++++++++++++++++ docs/level-24-files-over-http/config.yml | 2 ++ tests/test_docs.py | 5 +++ 6 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 docs/level-24-files-over-http/.moban.yml create mode 100644 docs/level-24-files-over-http/README.rst create mode 100644 docs/level-24-files-over-http/config.yml diff --git a/docs/README.rst b/docs/README.rst index e5792f2a..483590b5 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -26,6 +26,7 @@ This section covers the use cases for moban. Please check them out individually. #. `Template copying from a zip to a zip`_ #. `Intermeidate targets`_ #. `Mobanfile inheritance`_ +#. `Files over http(s)`_ .. _Jinja2 command line: level-1-jinja2-cli .. _Template inheritance: level-2-template-inheritance @@ -50,4 +51,4 @@ This section covers the use cases for moban. Please check them out individually. .. _Template copying from a zip to a zip: level-21-copy-templates-into-an-alien-file-system .. _Intermeidate targets: level-22-intermediate-targets .. _Mobanfile inheritance: level-23-inherit-organisational-moban-file - +.. _Files over http(s): level-24-files-over-http diff --git a/docs/index.rst b/docs/index.rst index b6cd4efe..1d420d57 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -38,6 +38,7 @@ examples folder. level-21-copy-templates-into-an-alien-file-system/README.rst level-22-intermediate-targets/README.rst level-23-inherit-organisational-moban-file/README.rst + level-24-files-over-http/README.rst For more complex use case, please look at `its usage in pyexcel project `_ diff --git a/docs/level-24-files-over-http/.moban.yml b/docs/level-24-files-over-http/.moban.yml new file mode 100644 index 00000000..60f9e142 --- /dev/null +++ b/docs/level-24-files-over-http/.moban.yml @@ -0,0 +1,9 @@ +configuration: + template_dir: + - "https://raw.githubusercontent.com/moremoban/pypi-mobans/dev/templates/" + - local + configuration: config.yml + configuration_dir: "https://raw.githubusercontent.com/moremoban/pypi-mobans/dev/config/" +targets: + - mytravis.yml: travis.yml.jj2 + - test.txt: demo.txt.jj2 diff --git a/docs/level-24-files-over-http/README.rst b/docs/level-24-files-over-http/README.rst new file mode 100644 index 00000000..00ae44a3 --- /dev/null +++ b/docs/level-24-files-over-http/README.rst @@ -0,0 +1,39 @@ +level 24: templates and configuration files over http(s) +================================================================================ + +.. note:: + + You will need to install httpfs + +Why not to take a template off the web? Once a template is written somewhere +by somebody, as long as it is good and useful, it is always to reuse it, +isn't it? DRY principle kicks in. + +Now with mobanfile, it is possible to package up your mobans/templates and +configuration files from a HTTP(S) protocol. + + +Here are the sample file:: + + configuration: + template_dir: + - "https://raw.githubusercontent.com/moremoban/pypi-mobans/dev/templates/" + - local + configuration: config.yml + configuration_dir: "https://raw.githubusercontent.com/moremoban/pypi-mobans/dev/config/" + targets: + - mytravis.yml: travis.yml.jj2 + - test.txt: demo.txt.jj2 + +When you refer to it in configuration section, here is the syntax:: + + configuration: + template_dir: + - "https://raw.githubusercontent.com/moremoban/pypi-mobans/dev/templates/" + +And the limitation is: you cannot have more than one http based url in the list +of `template_dir`. + +.. warn:: + + The trailing '/' must be there. diff --git a/docs/level-24-files-over-http/config.yml b/docs/level-24-files-over-http/config.yml new file mode 100644 index 00000000..cdb4f7dd --- /dev/null +++ b/docs/level-24-files-over-http/config.yml @@ -0,0 +1,2 @@ +overrides: data.yml +level24: "files over http protocol" diff --git a/tests/test_docs.py b/tests/test_docs.py index b0c735cb..bbfd821b 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -129,6 +129,11 @@ def test_level_9(self): folder = "level-9-moban-dependency-as-pypi-package" self.run_moban(["moban"], folder, [("test.txt", expected)]) + def test_level_24(self): + expected = "pypi-mobans: files over http protocol" + folder = "level-24-files-over-http" + self.run_moban(["moban"], folder, [("test.txt", expected)]) + def test_level_9_deprecated(self): expected = "pypi-mobans: moban dependency as pypi package" folder = "deprecated-level-9-moban-dependency-as-pypi-package" From c3b4ef439dd259f413fc9d981d015e5e2b4294ff Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 1 May 2020 22:31:53 +0100 Subject: [PATCH 08/14] :micrscope: more unit tests --- moban/core/hashstore.py | 27 +++++++++++++++++---------- moban/core/moban_factory.py | 4 +++- moban/exceptions.py | 4 ++++ moban/externals/file_system.py | 11 +++++++---- tests/test_hash_store.py | 20 +++++++++++++++++++- 5 files changed, 50 insertions(+), 16 deletions(-) diff --git a/moban/core/hashstore.py b/moban/core/hashstore.py index 0f3ddfbb..db428a22 100644 --- a/moban/core/hashstore.py +++ b/moban/core/hashstore.py @@ -1,9 +1,7 @@ import json import hashlib -from fs.errors import NoSysPath - -from moban import constants +from moban import constants, exceptions from moban.externals import file_system @@ -22,12 +20,14 @@ def __init__(self): self.hashes = {} def is_file_changed(self, file_name, file_content, source_template): - changed = self._is_source_updated( + changed, with_permission = self._is_source_updated( file_name, file_content, source_template ) if changed is False: - target_hash = get_file_hash(file_name) + target_hash = get_file_hash( + file_name, with_permission=with_permission + ) if target_hash != self.hashes[file_name]: changed = True return changed @@ -35,13 +35,17 @@ def is_file_changed(self, file_name, file_content, source_template): def _is_source_updated(self, file_name, file_content, source_template): changed = True content = file_content + with_permission = True try: content = _mix( file_content, oct(file_system.file_permissions(source_template)), ) - except NoSysPath: + except exceptions.NoPermissionsNeeded: # HttpFs does not have getsyspath + # zip, tar have no permission + # win32 does not work + with_permission = False pass content_hash = get_hash(content) if file_system.exists(file_name): @@ -53,7 +57,7 @@ def _is_source_updated(self, file_name, file_content, source_template): if changed: self.hashes[file_name] = content_hash - return changed + return changed, with_permission def save_hashes(self): with open(self.cache_file, "w") as f: @@ -63,12 +67,15 @@ def save_hashes(self): HASH_STORE = HashStore() -def get_file_hash(afile): +def get_file_hash(afile, with_permission=True): content = file_system.read_bytes(afile) try: - content = _mix(content, oct(file_system.file_permissions(afile))) - except NoSysPath: + if with_permission: + content = _mix(content, oct(file_system.file_permissions(afile))) + except exceptions.NoPermissionsNeeded: # HttpFs does not have getsyspath + # zip, tar have no permission + # win32 does not work pass return get_hash(content) diff --git a/moban/core/moban_factory.py b/moban/core/moban_factory.py index 1592ea4f..5db4b6b5 100644 --- a/moban/core/moban_factory.py +++ b/moban/core/moban_factory.py @@ -175,8 +175,10 @@ def apply_template(self, template_abs_path, template, data, output_file): file_system.file_permissions_copy( template_abs_path, output_file ) - except NoSysPath: + except exceptions.NoPermissionsNeeded: # HttpFs does not have getsyspath + # zip, tar have no permission + # win32 does not work pass return flag except exceptions.FileNotFound: diff --git a/moban/exceptions.py b/moban/exceptions.py index b3907dfc..aa58149b 100644 --- a/moban/exceptions.py +++ b/moban/exceptions.py @@ -32,3 +32,7 @@ class NoGitCommand(Exception): class UnsupportedPyFS2Protocol(Exception): pass + + +class NoPermissionsNeeded(Exception): + pass diff --git a/moban/externals/file_system.py b/moban/externals/file_system.py index e890fd47..6639f98c 100644 --- a/moban/externals/file_system.py +++ b/moban/externals/file_system.py @@ -224,12 +224,15 @@ def file_permissions(url): if not exists(url): raise exceptions.FileNotFound(url) if sys.platform == "win32": - return 0 + raise exceptions.NoPermissionsNeeded() elif is_zip_alike_url(url): - return 755 + raise exceptions.NoPermissionsNeeded() else: - unix_path = system_path(url) - return stat.S_IMODE(os.stat(unix_path).st_mode) + try: + unix_path = system_path(url) + return stat.S_IMODE(os.stat(unix_path).st_mode) + except fs.errors.NoSysPath: + raise exceptions.NoPermissionsNeeded() def url_split(url): diff --git a/tests/test_hash_store.py b/tests/test_hash_store.py index 3220fee5..0f098d64 100644 --- a/tests/test_hash_store.py +++ b/tests/test_hash_store.py @@ -1,10 +1,12 @@ import os import sys +from unittest.mock import patch from nose import SkipTest from moban.externals import file_system -from moban.core.hashstore import HashStore +from moban.exceptions import NoPermissionsNeeded +from moban.core.hashstore import HashStore, get_file_hash class TestHashStore: @@ -17,6 +19,7 @@ def setUp(self): "test content".encode("utf-8"), self.source_template, ) + self.file_hash = get_file_hash(self.source_template) def tearDown(self): if os.path.exists(".moban.hashes"): @@ -25,6 +28,21 @@ def tearDown(self): def test_simple_use_case(self): hs = HashStore() flag = hs.is_file_changed(*self.fixture) + + hs.save_hashes() + assert flag is True + + @patch("moban.core.hashstore.file_system.file_permissions") + def test_permission_check_failed(self, fake): + """ + when system permission fails, both source hash and + target hash shall not use permission. + """ + fake.side_effect = [NoPermissionsNeeded()] + hs = HashStore() + flag = hs.is_file_changed(*self.fixture) + + assert hs.hashes["test.out"] != self.file_hash hs.save_hashes() assert flag is True From d3645ba766be3019e7f5d561fdb9a5a62b15f0cb Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 1 May 2020 23:10:23 +0100 Subject: [PATCH 09/14] :books: update version and documentation --- .moban.cd/moban.yml | 4 ++-- docs/conf.py | 2 +- docs/level-24-files-over-http/README.rst | 21 ++++++++++++++++++--- moban/_version.py | 2 +- moban/core/mobanfile/__init__.py | 2 ++ moban/exceptions.py | 4 ++++ moban/main.py | 1 + setup.py | 2 +- 8 files changed, 30 insertions(+), 8 deletions(-) diff --git a/.moban.cd/moban.yml b/.moban.cd/moban.yml index 6d6cb240..6c50d101 100644 --- a/.moban.cd/moban.yml +++ b/.moban.cd/moban.yml @@ -4,8 +4,8 @@ organisation: moremoban author: C. W. contact: wangc_2011@hotmail.com license: MIT -version: 0.7.1 -current_version: 0.7.1 +version: 0.7.2 +current_version: 0.7.2 release: 0.7.1 branch: master master: index diff --git a/docs/conf.py b/docs/conf.py index 7f30573d..3e24c6c1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -25,7 +25,7 @@ copyright = '2017-2020 Onni Software Ltd.' author = 'C. W.' # The short X.Y version -version = '0.7.1' +version = '0.7.2' # The full version, including alpha/beta/rc tags release = '0.7.1' diff --git a/docs/level-24-files-over-http/README.rst b/docs/level-24-files-over-http/README.rst index 00ae44a3..86e8c07c 100644 --- a/docs/level-24-files-over-http/README.rst +++ b/docs/level-24-files-over-http/README.rst @@ -31,9 +31,24 @@ When you refer to it in configuration section, here is the syntax:: template_dir: - "https://raw.githubusercontent.com/moremoban/pypi-mobans/dev/templates/" -And the limitation is: you cannot have more than one http based url in the list -of `template_dir`. - .. warn:: The trailing '/' must be there. + + +Maintenance note +-------------------------------------------------------------------------------- + +To the maintainer, in order to eat the dog food. Please checkout pypi-mobans +and run a http server inside local pypi-mobans folder. + +Then update moban's mobanfile to:: + + configuration: + template_dir: + - "http://localhost:8000/templates/" + - "http://localhost:8000/statics/" + - ".moban.d" + +Then run `make update` + diff --git a/moban/_version.py b/moban/_version.py index 2a8eb359..233ef58a 100644 --- a/moban/_version.py +++ b/moban/_version.py @@ -1,2 +1,2 @@ -__version__ = "0.7.1" +__version__ = "0.7.2" __author__ = "C. W." diff --git a/moban/core/mobanfile/__init__.py b/moban/core/mobanfile/__init__.py index 34d3f921..6d875d67 100644 --- a/moban/core/mobanfile/__init__.py +++ b/moban/core/mobanfile/__init__.py @@ -45,6 +45,7 @@ def handle_moban_file_v1(moban_file_configurations, command_line_options): moban_file_configurations[constants.LABEL_CONFIG], ) merged_options = merge(command_line_options, constants.DEFAULT_OPTIONS) + plugins_dirs = merged_options.get(constants.LABEL_PLUGIN_DIRS) if plugins_dirs: handle_plugin_dirs(plugins_dirs) @@ -66,6 +67,7 @@ def handle_moban_file_v1(moban_file_configurations, command_line_options): ) ) ) + extensions = moban_file_configurations.get(constants.LABEL_EXTENSIONS) if extensions: core.ENGINES.register_extensions(extensions) diff --git a/moban/exceptions.py b/moban/exceptions.py index aa58149b..fcb96e60 100644 --- a/moban/exceptions.py +++ b/moban/exceptions.py @@ -36,3 +36,7 @@ class UnsupportedPyFS2Protocol(Exception): class NoPermissionsNeeded(Exception): pass + + +class SingleHTTPURLConstraint(Exception): + pass diff --git a/moban/main.py b/moban/main.py index 379dd8e7..ef2f9bc6 100644 --- a/moban/main.py +++ b/moban/main.py @@ -43,6 +43,7 @@ def main(): options[constants.EXTENSION_DICT] = handle_custom_extensions( options.pop(constants.LABEL_EXTENSION) ) + OPTIONS.update(options) moban_file = options[constants.LABEL_MOBANFILE] if moban_file is None: diff --git a/setup.py b/setup.py index 72af6c1f..adfdf90c 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ NAME = "moban" AUTHOR = "C. W." -VERSION = "0.7.1" +VERSION = "0.7.2" EMAIL = "wangc_2011@hotmail.com" LICENSE = "MIT" ENTRY_POINTS = { From 141f8506ad67c85d5756a69112483c1569f4ee7e Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 1 May 2020 23:14:10 +0100 Subject: [PATCH 10/14] :fire: remove unused import --- moban/core/moban_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moban/core/moban_factory.py b/moban/core/moban_factory.py index 5db4b6b5..df569903 100644 --- a/moban/core/moban_factory.py +++ b/moban/core/moban_factory.py @@ -3,7 +3,7 @@ import logging from collections import defaultdict -from fs.errors import NoSysPath, ResourceNotFound +from fs.errors import ResourceNotFound from lml.plugin import PluginManager from moban import constants, exceptions From 1780ffd43ddfe6d209c74ec48bd874ffecd9fca8 Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 1 May 2020 23:16:23 +0100 Subject: [PATCH 11/14] :fire: remove unused codes --- moban/core/moban_factory.py | 2 -- moban/main.py | 6 +----- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/moban/core/moban_factory.py b/moban/core/moban_factory.py index df569903..5b4ad8c5 100644 --- a/moban/core/moban_factory.py +++ b/moban/core/moban_factory.py @@ -1,5 +1,4 @@ import os -import sys import logging from collections import defaultdict @@ -16,7 +15,6 @@ from moban.externals.buffered_writer import BufferedWriter LOG = logging.getLogger(__name__) -PY3_ABOVE = sys.version_info[0] > 2 class MobanFactory(PluginManager): diff --git a/moban/main.py b/moban/main.py index ef2f9bc6..4890a458 100644 --- a/moban/main.py +++ b/moban/main.py @@ -12,6 +12,7 @@ import logging import argparse import logging.config +from io import StringIO from collections import defaultdict from ruamel.yaml import YAML @@ -22,11 +23,6 @@ from moban.externals import reporter, file_system from moban.program_options import OPTIONS -try: - from cStringIO import StringIO -except ImportError: - from io import StringIO - LOG = logging.getLogger() LOG_LEVEL = [logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG] From 4de4a5ad8bca6537d18034a42736b031f37038d9 Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 1 May 2020 23:35:50 +0100 Subject: [PATCH 12/14] :books: yaml > allows a v long string with new line. >- does the same but no new line --- docs/level-24-files-over-http/.moban.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/level-24-files-over-http/.moban.yml b/docs/level-24-files-over-http/.moban.yml index 60f9e142..24b9565a 100644 --- a/docs/level-24-files-over-http/.moban.yml +++ b/docs/level-24-files-over-http/.moban.yml @@ -3,7 +3,8 @@ configuration: - "https://raw.githubusercontent.com/moremoban/pypi-mobans/dev/templates/" - local configuration: config.yml - configuration_dir: "https://raw.githubusercontent.com/moremoban/pypi-mobans/dev/config/" + configuration_dir: >- + https://raw.githubusercontent.com/moremoban/pypi-mobans/dev/config/ targets: - mytravis.yml: travis.yml.jj2 - test.txt: demo.txt.jj2 From 838378616d59f5432729c38fb2c11cc1149cdf97 Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 1 May 2020 23:53:38 +0100 Subject: [PATCH 13/14] :eggs: :ferris_wheel: ready to release --- .moban.cd/changelog.yml | 4 ++-- .moban.cd/moban.yml | 2 +- CHANGELOG.rst | 6 +++--- README.rst | 4 ++++ docs/conf.py | 2 +- setup.py | 6 +++--- tests/requirements.txt | 1 + 7 files changed, 15 insertions(+), 10 deletions(-) diff --git a/.moban.cd/changelog.yml b/.moban.cd/changelog.yml index df66c6a8..b0a6f2ba 100644 --- a/.moban.cd/changelog.yml +++ b/.moban.cd/changelog.yml @@ -4,8 +4,8 @@ releases: - changes: - action: Added details: - - "Partial support for templates and configuration files over HTTP(S) protocol with httpfs." - date: tbd + - "Support for templates and configuration files over HTTP(S) protocol with httpfs! Yepee!" + date: 1.5.2020 version: 0.7.2 - changes: - action: Fixed diff --git a/.moban.cd/moban.yml b/.moban.cd/moban.yml index 6c50d101..9dfc9851 100644 --- a/.moban.cd/moban.yml +++ b/.moban.cd/moban.yml @@ -6,7 +6,7 @@ contact: wangc_2011@hotmail.com license: MIT version: 0.7.2 current_version: 0.7.2 -release: 0.7.1 +release: 0.7.2 branch: master master: index command_line_interface: "moban" diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b8663327..c9e63b0a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,13 +1,13 @@ Change log ================================================================================ -0.7.2 - tbd +0.7.2 - 1.5.2020 -------------------------------------------------------------------------------- **Added** -#. Partial support for templates and configuration files over HTTP(S) protocol - with httpfs. +#. Support for templates and configuration files over HTTP(S) protocol with + httpfs! Yepee! 0.7.1 - 25.04.2020 -------------------------------------------------------------------------------- diff --git a/README.rst b/README.rst index ca15aa7c..159fea4d 100644 --- a/README.rst +++ b/README.rst @@ -173,6 +173,10 @@ Templates and configuration files over HTTP(S) -o _version.py +In an edge case, if github repo's public url is given, +this github repo shall not have sub repos. This library will fail to +translate sub-repo as url. No magic. + Templates and configuration files in a git repo ================================================================================ diff --git a/docs/conf.py b/docs/conf.py index 3e24c6c1..4331c1ce 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -27,7 +27,7 @@ # The short X.Y version version = '0.7.2' # The full version, including alpha/beta/rc tags -release = '0.7.1' +release = '0.7.2' # -- General configuration --------------------------------------------------- diff --git a/setup.py b/setup.py index adfdf90c..5ef79b7a 100644 --- a/setup.py +++ b/setup.py @@ -53,7 +53,7 @@ "General purpose static text generator" ) URL = "https://github.com/moremoban/moban" -DOWNLOAD_URL = "%s/archive/0.7.1.tar.gz" % URL +DOWNLOAD_URL = "%s/archive/0.7.2.tar.gz" % URL FILES = ["README.rst", "CONTRIBUTORS.rst", "CHANGELOG.rst"] KEYWORDS = [ "python", @@ -97,8 +97,8 @@ } # You do not need to read beyond this line PUBLISH_COMMAND = "{0} setup.py sdist bdist_wheel upload -r pypi".format(sys.executable) -GS_COMMAND = ("gs moban v0.7.1 " + - "Find 0.7.1 in changelog for more details") +GS_COMMAND = ("gs moban v0.7.2 " + + "Find 0.7.2 in changelog for more details") NO_GS_MESSAGE = ("Automatic github release is disabled. " + "Please install gease to enable it.") UPLOAD_FAILED_MSG = ( diff --git a/tests/requirements.txt b/tests/requirements.txt index 4979027e..093c2e70 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -15,3 +15,4 @@ jinja2_time pypifs gitfs2 jinja2-python-version>=1.1.2 +httpfs From eb06917d6e7563167b71a1b3d1e462bf8406a82d Mon Sep 17 00:00:00 2001 From: chfw Date: Fri, 1 May 2020 23:56:07 +0100 Subject: [PATCH 14/14] :newspaper: add missing test fixture --- docs/level-24-files-over-http/local/demo.txt.jj2 | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/level-24-files-over-http/local/demo.txt.jj2 diff --git a/docs/level-24-files-over-http/local/demo.txt.jj2 b/docs/level-24-files-over-http/local/demo.txt.jj2 new file mode 100644 index 00000000..8c478790 --- /dev/null +++ b/docs/level-24-files-over-http/local/demo.txt.jj2 @@ -0,0 +1 @@ +{{name}}: {{level24}} \ No newline at end of file