From adf39cdf21f0bc929040f3732c4273a2bb920501 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Tue, 20 Oct 2015 14:54:16 +0200 Subject: [PATCH 01/66] maintenance branch is now 1.2.8-dev --- .versioneer-lookup | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.versioneer-lookup b/.versioneer-lookup index 41fd3a301..a6d430799 100644 --- a/.versioneer-lookup +++ b/.versioneer-lookup @@ -10,10 +10,10 @@ # master shall not use the lookup table, only tags master -# maintenance is currently the branch for preparation of maintenance release 1.2.7 +# maintenance is currently the branch for preparation of maintenance release 1.2.8 # so are any fix/... branches -maintenance 1.2.7-dev 536bb31965db17b969e7c1c53e241ddac4ae1814 -fix/.* 1.2.7-dev 536bb31965db17b969e7c1c53e241ddac4ae1814 +maintenance 1.2.8-dev 6c622f7c4332b71c6ece59552ffc87c146155c84 +fix/.* 1.2.8-dev 6c622f7c4332b71c6ece59552ffc87c146155c84 # Special case disconnected checkouts, e.g. 'git checkout ' \(detached.* From 57bf457d973307f92e93bb335eada4a1bb795e86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Thu, 22 Oct 2015 13:42:57 +0200 Subject: [PATCH 02/66] Fix: current filename in job data should never be prefixed with / That was a tiny detail missing in 8bef18c2a82a70b7be968908534d68d6bea16f39 that caused issues with detecting which file was currently selected on the printer's SD card (for "do not delete that" checks on the API). --- src/octoprint/printer/standard.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/octoprint/printer/standard.py b/src/octoprint/printer/standard.py index 8e11b6421..8b909705c 100644 --- a/src/octoprint/printer/standard.py +++ b/src/octoprint/printer/standard.py @@ -645,6 +645,8 @@ def _setJobData(self, filename, filesize, sd): if filename is not None: if sd: path_in_storage = filename + if path_in_storage.startswith("/"): + path_in_storage = path_in_storage[1:] path_on_disk = None else: path_in_storage = self._fileManager.path_in_storage(FileDestinations.LOCAL, filename) From 9ce9af28c3c5b79403429319328f93a12defbb7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Mon, 29 Jun 2015 17:27:15 +0200 Subject: [PATCH 03/66] Updating versioneer & switch to PEP440 versioning (cherry picked from commit 137db0b) --- .versioneer-lookup | 2 +- setup.cfg | 13 + setup.py | 8 - src/octoprint/_version.py | 605 ++++++++--- versioneer.py | 2154 ++++++++++++++++++++++++++----------- 5 files changed, 1983 insertions(+), 799 deletions(-) create mode 100644 setup.cfg diff --git a/.versioneer-lookup b/.versioneer-lookup index a6d430799..78d848c94 100644 --- a/.versioneer-lookup +++ b/.versioneer-lookup @@ -19,4 +19,4 @@ fix/.* 1.2.8-dev 6c622f7c4332b71c6ece59552ffc87c146155c84 \(detached.* # every other branch is a development branch and thus gets resolved to 1.3.0-dev for now -.* 1.3.0-dev 198d3450d94be1a2 +.* 1.3.0 198d3450d94be1a2 pep440-pre diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 000000000..e814cb446 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,13 @@ + +# See the docstring in versioneer.py for instructions. Note that you must +# re-run 'versioneer.py setup' after changing this section, and commit the +# resulting files. + +[versioneer] +VCS = git +style = pep440 +versionfile_source = src/octoprint/_version.py +versionfile_build = octoprint/_version.py +tag_prefix = +parentdir_prefix = +lookupfile = .versioneer-lookup diff --git a/setup.py b/setup.py index 80ed83c4b..878c18b4e 100644 --- a/setup.py +++ b/setup.py @@ -59,14 +59,6 @@ # Dependency links for any of the aforementioned dependencies DEPENDENCY_LINKS = [] -# Versioneer configuration -versioneer.VCS = 'git' -versioneer.versionfile_source = 'src/octoprint/_version.py' -versioneer.versionfile_build = 'octoprint/_version.py' -versioneer.tag_prefix = '' -versioneer.parentdir_prefix = '' -versioneer.lookupfile = '.versioneer-lookup' - #----------------------------------------------------------------------------------------------------------------------- # Anything below here is just command setup and general setup configuration diff --git a/src/octoprint/_version.py b/src/octoprint/_version.py index 35090f5b8..b96249a5b 100644 --- a/src/octoprint/_version.py +++ b/src/octoprint/_version.py @@ -1,28 +1,81 @@ # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (build by setup.py sdist) and build +# feature). Distribution tarballs (built by setup.py sdist) and build # directories (produced by setup.py build) will contain a much shorter file # that just contains the computed version number. # This file is released into the public domain. Generated by -# versioneer-0.10 (https://github.com/warner/python-versioneer) - -# these strings will be replaced by git during git-archive -git_refnames = "$Format:%d$" -git_full = "$Format:%H$" +# versioneer-0.15+dev (https://github.com/warner/python-versioneer) +"""Git implementation of _version.py.""" +import errno +import os +import re import subprocess import sys -import errno + + +def get_keywords(): + """Get the keywords needed to look up the version information.""" + # these strings will be replaced by git during git-archive. + # setup.py/versioneer.py will grep for the variable names, so they must + # each be defined on a line of their own. _version.py will just call + # get_keywords(). + git_refnames = "$Format:%d$" + git_full = "$Format:%H$" + keywords = {"refnames": git_refnames, "full": git_full} + return keywords + + +class VersioneerConfig: + + """Container for Versioneer configuration parameters.""" + + +def get_config(): + """Create, populate and return the VersioneerConfig() object.""" + # these strings are filled in when 'setup.py versioneer' creates + # _version.py + cfg = VersioneerConfig() + cfg.VCS = "git" + cfg.style = "pep440" + cfg.tag_prefix = "" + cfg.parentdir_prefix = "" + cfg.versionfile_source = "src/octoprint/_version.py" + cfg.lookupfile = None + cfg.verbose = False + return cfg + + +class NotThisMethod(Exception): + + """Exception raised if a method is not valid for the current scenario.""" + + +LONG_VERSION_PY = {} +HANDLERS = {} + + +def register_vcs_handler(vcs, method): # decorator + """Decorator to mark a method as the handler for a particular VCS.""" + def decorate(f): + """Store f in HANDLERS[vcs][method].""" + if vcs not in HANDLERS: + HANDLERS[vcs] = {} + HANDLERS[vcs][method] = f + return f + return decorate def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): + """Call the given command(s).""" assert isinstance(commands, list) p = None for c in commands: try: + dispcmd = str([c] + args) # remember shell=False, so use git.cmd on windows, not just git p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE, stderr=(subprocess.PIPE if hide_stderr @@ -33,7 +86,7 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): if e.errno == errno.ENOENT: continue if verbose: - print("unable to run %s" % args[0]) + print("unable to run %s" % dispcmd) print(e) return None else: @@ -41,59 +94,67 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): print("unable to find command, tried %s" % (commands,)) return None stdout = p.communicate()[0].strip() - if sys.version >= '3': + if sys.version_info[0] >= 3: stdout = stdout.decode() if p.returncode != 0: if verbose: - print("unable to run %s (error)" % args[0]) + print("unable to run %s (error)" % dispcmd) return None return stdout -import sys -import re -import os.path +def versions_from_parentdir(parentdir_prefix, root, verbose): + """Try to determine the version from the parent directory name. -def get_gits(root, verbose=False): - if not os.path.exists(os.path.join(root, ".git")): + Source tarballs conventionally unpack into a directory that includes + both the project name and a version string. + """ + dirname = os.path.basename(root) + if not dirname.startswith(parentdir_prefix): if verbose: - print("no .git in %s" % root) - return None - - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - return GITS + print("guessing rootdir is '%s', but '%s' doesn't start with " + "prefix '%s'" % (root, dirname, parentdir_prefix)) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None} -def get_expanded_variables(versionfile_abs): +@register_vcs_handler("git", "get_keywords") +def git_get_keywords(versionfile_abs): + """Extract version information from the given file.""" # the code embedded in _version.py can just fetch the value of these - # variables. When used from setup.py, we don't want to import - # _version.py, so we do it with a regexp instead. This function is not - # used from _version.py. - variables = {} + # keywords. When used from setup.py, we don't want to import _version.py, + # so we do it with a regexp instead. This function is not used from + # _version.py. + keywords = {} try: - f = open(versionfile_abs,"r") + f = open(versionfile_abs, "r") for line in f.readlines(): if line.strip().startswith("git_refnames ="): mo = re.search(r'=\s*"(.*)"', line) if mo: - variables["refnames"] = mo.group(1) + keywords["refnames"] = mo.group(1) if line.strip().startswith("git_full ="): mo = re.search(r'=\s*"(.*)"', line) if mo: - variables["full"] = mo.group(1) + keywords["full"] = mo.group(1) f.close() except EnvironmentError: pass - return variables + return keywords -def versions_from_expanded_variables(variables, tag_prefix, verbose=False): - refnames = variables["refnames"].strip() + +@register_vcs_handler("git", "keywords") +def git_versions_from_keywords(keywords, tag_prefix, verbose): + """Get version information from git keywords.""" + if not keywords: + raise NotThisMethod("no keywords at all, weird") + refnames = keywords["refnames"].strip() if refnames.startswith("$Format"): if verbose: - print("variables are unexpanded, not using") - return {} # unexpanded, so not in an unpacked git-archive tarball + print("keywords are unexpanded, not using") + raise NotThisMethod("unexpanded keywords, not a git-archive tarball") refs = set([r.strip() for r in refnames.strip("()").split(",")]) # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. @@ -110,6 +171,14 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False): tags = set([r for r in refs if re.search(r'\d', r)]) if verbose: print("discarding '%s', no digits" % ",".join(refs-tags)) + + branches = [r for r in refs if not r.startswith(TAG) and r != "HEAD" and not r.startswith("refs/")] + if verbose: + print("likely branches: %s" % ",".join(sorted(branches))) + branch = None + if branches: + branch = branches[0] + if verbose: print("likely tags: %s" % ",".join(sorted(tags))) for ref in sorted(tags): @@ -118,161 +187,413 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False): r = ref[len(tag_prefix):] if verbose: print("picking %s" % r) - return { "version": r, - "full": variables["full"].strip() } - # no suitable tags, so we use the full revision id + + result = {"version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None + } + if branch is not None: + result["branch"] = branch + return result + # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: - print("no suitable tags, using full revision id") - return { "version": variables["full"].strip(), - "full": variables["full"].strip() } + print("no suitable tags, using unknown + full revision id") + return {"version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": "no suitable tags"} + + +@register_vcs_handler("git", "pieces_from_vcs") +def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): + """Get version from 'git describe' in the root of the source tree. + + This only gets called if the git-archive 'subst' keywords were *not* + expanded, and _version.py hasn't already been rewritten with a short + version string, meaning we're inside a checked out source tree. + """ + if not os.path.exists(os.path.join(root, ".git")): + if verbose: + print("no .git in %s" % root) + raise NotThisMethod("no .git directory") + + GITS = ["git"] + if sys.platform == "win32": + GITS = ["git.cmd", "git.exe"] + # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] + # if there isn't one, this yields HEX[-dirty] (no NUM) + describe_out = run_command(GITS, ["describe", "--tags", "--dirty", + "--always", "--long", + "--match", "%s*" % tag_prefix], + cwd=root) + # --long was added in git-1.5.5 + if describe_out is None: + raise NotThisMethod("'git describe' failed") + describe_out = describe_out.strip() + full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + if full_out is None: + raise NotThisMethod("'git rev-parse' failed") + full_out = full_out.strip() + + pieces = {} + pieces["long"] = full_out + pieces["short"] = full_out[:7] # maybe improved later + pieces["error"] = None + + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] + # TAG might have hyphens. + git_describe = describe_out + + # look for -dirty suffix + dirty = git_describe.endswith("-dirty") + pieces["dirty"] = dirty + if dirty: + git_describe = git_describe[:git_describe.rindex("-dirty")] + + # figure out our branch + abbrev_ref_out = run_command(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], cwd=root) + if abbrev_ref_out is not None: + pieces["branch"] = abbrev_ref_out.strip() + + # now we have TAG-NUM-gHEX or HEX + + if "-" in git_describe: + # TAG-NUM-gHEX + mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + if not mo: + # unparseable. Maybe git-describe is misbehaving? + pieces["error"] = ("unable to parse git-describe output: '%s'" + % describe_out) + return pieces + + # tag + full_tag = mo.group(1) + if not full_tag.startswith(tag_prefix): + if verbose: + fmt = "tag '%s' doesn't start with prefix '%s'" + print(fmt % (full_tag, tag_prefix)) + pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" + % (full_tag, tag_prefix)) + return pieces + pieces["closest-tag"] = full_tag[len(tag_prefix):] + + # distance: number of commits since tag + pieces["distance"] = int(mo.group(2)) + + # commit: short hex revision ID + pieces["short"] = mo.group(3) + + else: + # HEX: no tags + pieces["closest-tag"] = None + count_out = run_command(GITS, ["rev-list", "HEAD", "--count"], + cwd=root) + pieces["distance"] = int(count_out) # total number of commits + + return pieces + + +@register_vcs_handler("git", "parse_lookup_file") +def git_parse_lookup_file(path): + if not os.path.exists(path): + return [] + + import re + lookup = [] + with open(path, "r") as f: + for line in f: + if '#' in line: + line = line[:line.rindex("#")] + line = line.strip() + try: + split_line = line.split() + if len(split_line) == 3: + pattern, tag, ref_commit = split_line + lookup.append([re.compile(pattern), tag, ref_commit, None]) + elif len(split_line) == 4: + pattern, tag, ref_commit, render = split_line + lookup.append([re.compile(pattern), tag, ref_commit, render]) + elif len(split_line) >= 1: + lookup.append([re.compile(split_line[0]), None, None, None]) + except: + break + return lookup -def versions_from_lookup(lookup, root, verbose=False): - GITS = get_gits(root, verbose=verbose) - if GITS is None: - return {} +@register_vcs_handler("git", "pieces_from_lookup") +def git_pieces_from_lookup(lookup, root, verbose, run_command=run_command): + GITS = ["git"] + if sys.platform == "win32": + GITS = ["git.cmd", "git.exe"] stdout = run_command(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], cwd=root) if stdout is None: - return {} + raise NotThisMethod("git rev-parse --abbrev-ref HEAD failed") current_branch = stdout.strip() - for matcher, tag, ref_commit in lookup: + for matcher, tag, ref_commit, render in lookup: if matcher.match(current_branch): if tag is None or ref_commit is None: - return {} + raise NotThisMethod("tag or ref_commit is unset for this branch") stdout = run_command(GITS, ["rev-list", "%s..HEAD" % ref_commit, "--count"], cwd=root) if stdout is None: - return {} - num_commits = stdout.strip() + raise NotThisMethod("git rev-list %s..HEAD --count failed" % ref_commit) + try: + num_commits = int(stdout.strip()) + except ValueError: + raise NotThisMethod("git rev-list %s..HEAD --count didn't return a valid number" % ref_commit) stdout =run_command(GITS, ["rev-parse", "--short", "HEAD"], cwd=root) if stdout is None: - return {} + raise NotThisMethod("git describe rev-parse --short HEAD failed") short_hash = stdout.strip() stdout = run_command(GITS, ["describe", "--tags", "--dirty", "--always"], cwd=root) if stdout is None: - return {} + raise NotThisMethod("git describe --tags --dirty --always failed") dirty = stdout.strip().endswith("-dirty") stdout = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) if stdout is None: - return {} + raise NotThisMethod("git rev-parse HEAD failed") full = stdout.strip() - version = "%s-%s-g%s" % (tag, num_commits, short_hash) - if dirty: - version += "-dirty" - full += "-dirty" - return {"version": version, "full": full, "branch": current_branch} - - return {} + return { + "long": full, + "short": short_hash, + "dirty": dirty, + "branch": current_branch, + "closest-tag": tag, + "distance": num_commits, + "error": None, + "render": render + } + + raise NotThisMethod("no matching lookup definition found") + + +def plus_or_dot(pieces): + """Return a + if we don't already have one, else return a .""" + if "+" in pieces.get("closest-tag", ""): + return "." + return "+" + + +def render_pep440(pieces): + """Build up version string, with post-release "local version identifier". + + Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you + get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + + Exceptions: + 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += plus_or_dot(pieces) + rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0+untagged.%d.g%s" % (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_pre(pieces): + """TAG[.post.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post.devDISTANCE + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += ".post.dev%d" % pieces["distance"] + else: + # exception #1 + rendered = "0.post.dev%d" % pieces["distance"] + return rendered + + +def render_pep440_post(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX] . + + The ".dev0" means dirty. Note that .dev0 sorts backwards + (a dirty tree will appear "older" than the corresponding clean one), + but you shouldn't be releasing software with -dirty anyways. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%s" % pieces["short"] + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += "+g%s" % pieces["short"] + return rendered + + +def render_pep440_old(pieces): + """TAG[.postDISTANCE[.dev0]] . + + The ".dev0" means dirty. + + Eexceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + return rendered -def versions_from_vcs(tag_prefix, root, verbose=False): - # this runs 'git' from the root of the source tree. This only gets called - # if the git-archive 'subst' variables were *not* expanded, and - # _version.py hasn't already been rewritten with a short version string, - # meaning we're inside a checked out source tree. - GITS = get_gits(root, verbose=verbose) - if GITS is None: - return {} +def render_git_describe(pieces): + """TAG[-DISTANCE-gHEX][-dirty]. - stdout = run_command(GITS, ["describe", "--tags", "--dirty", "--always"], - cwd=root) - if stdout is None: - return {} - if not stdout.startswith(tag_prefix): - if verbose: - print("tag '%s' doesn't start with prefix '%s'" % (stdout, tag_prefix)) - return {} - tag = stdout[len(tag_prefix):] - stdout = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) - if stdout is None: - return {} - full = stdout.strip() - if tag.endswith("-dirty"): - full += "-dirty" + Like 'git describe --tags --dirty --always'. - stdout = run_command(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], - cwd=root) - if stdout is None: - branch = None + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) else: - branch = stdout.strip() - return {"version": tag, "full": full, "branch": branch} + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered -def versions_from_parentdir(parentdir_prefix, root, verbose=False): - # Source tarballs conventionally unpack into a directory that includes - # both the project name and a version string. - dirname = os.path.basename(root) - if not dirname.startswith(parentdir_prefix): - if verbose: - print("guessing rootdir is '%s', but '%s' doesn't start with prefix '%s'" % - (root, dirname, parentdir_prefix)) - return None - return {"version": dirname[len(parentdir_prefix):], "full": "", "branch": ""} +def render_git_describe_long(pieces): + """TAG-DISTANCE-gHEX[-dirty]. -tag_prefix = "" -parentdir_prefix = "" -versionfile_source = "src/octoprint/_version.py" -lookupfile = ".versioneer-lookup" + Like 'git describe --tags --dirty --always -long'. + The distance/hash is unconditional. -def parse_lookup_file(root, lookup_path=None): - if not lookup_path: - lookup_path = lookupfile - if not lookup_path: - return [] + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render(pieces, style): + """Render the given version pieces into the requested style.""" + if pieces["error"]: + return {"version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"]} + + if "render" in pieces and pieces["render"] is not None: + style = pieces["render"] + + if not style or style == "default": + style = "pep440" # the default + + if style == "pep440": + rendered = render_pep440(pieces) + elif style == "pep440-pre": + rendered = render_pep440_pre(pieces) + elif style == "pep440-post": + rendered = render_pep440_post(pieces) + elif style == "pep440-old": + rendered = render_pep440_old(pieces) + elif style == "git-describe": + rendered = render_git_describe(pieces) + elif style == "git-describe-long": + rendered = render_git_describe_long(pieces) + else: + raise ValueError("unknown style '%s'" % style) - path = os.path.join(root, lookup_path) - if not os.path.exists(path): - return [] + result = {"version": rendered, "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], "error": None} + if "branch" in pieces and pieces["branch"] is not None: + result["branch"] = pieces["branch"] + return result - import re - lookup = [] - with open(os.path.join(root, lookup_path), "r") as f: - for line in f: - if '#' in line: - line = line[:line.rindex('#')] - line = line.strip() - try: - split_line = line.split() - if len(split_line) == 3: - pattern, tag, ref_commit = split_line - lookup.append([re.compile(pattern), tag, ref_commit]) - elif len(split_line) >= 1: - lookup.append([re.compile(split_line[0]), None, None]) - except: - break - return lookup -def get_versions(default={"version": "unknown", "full": "", "branch": "unknown"}, lookup_path=None, verbose=False): +def get_versions(): + """Get version information or return default if unable to do so.""" # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have # __file__, we can work backwards from there to the root. Some # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which - # case we can only use expanded variables. + # case we can only use expanded keywords. - variables = { "refnames": git_refnames, "full": git_full } - ver = versions_from_expanded_variables(variables, tag_prefix, verbose) - if ver: - return ver + cfg = get_config() + verbose = cfg.verbose + + try: + return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, + verbose) + except NotThisMethod: + pass try: - root = os.path.abspath(__file__) + root = os.path.realpath(__file__) # versionfile_source is the relative path from the top of the source # tree (where the .git directory might live) to this file. Invert # this to find the root from __file__. - for i in range(len(versionfile_source.split("/"))): + for i in cfg.versionfile_source.split('/'): root = os.path.dirname(root) except NameError: - return default + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to find root of source tree"} - lookup = parse_lookup_file(root, lookup_path=lookup_path) - return (versions_from_lookup(lookup, root, verbose) - or versions_from_vcs(tag_prefix, root, verbose) - or versions_from_parentdir(parentdir_prefix, root, verbose) - or default) + lookupfile = os.path.join(root, cfg.lookupfile if cfg.lookupfile is not None else ".versioneer-lookup") + if os.path.exists(lookupfile): + try: + lookup_data = git_parse_lookup_file(lookupfile) + pieces = git_pieces_from_lookup(lookup_data, root, verbose) + return render(pieces, cfg.style) + except NotThisMethod: + pass + + try: + pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) + return render(pieces, cfg.style) + except NotThisMethod: + pass + + try: + if cfg.parentdir_prefix: + return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) + except NotThisMethod: + pass + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to compute version"} diff --git a/versioneer.py b/versioneer.py index 27e1741c2..51c6327de 100644 --- a/versioneer.py +++ b/versioneer.py @@ -1,5 +1,5 @@ -# Version: 0.10 +# Version: 0.15+dev """ The Versioneer @@ -9,9 +9,13 @@ * https://github.com/warner/python-versioneer * Brian Warner * License: Public Domain -* Compatible With: python2.6, 2.7, and 3.2, 3.3 - -[![Build Status](https://travis-ci.org/warner/python-versioneer.png?branch=master)](https://travis-ci.org/warner/python-versioneer) +* Compatible With: python2.6, 2.7, 3.2, 3.3, 3.4, and pypy +* [![Latest Version] +(https://pypip.in/version/versioneer/badge.svg?style=flat) +](https://pypi.python.org/pypi/versioneer/) +* [![Build Status] +(https://travis-ci.org/warner/python-versioneer.png?branch=master) +](https://travis-ci.org/warner/python-versioneer) This is a tool for managing a recorded version number in distutils-based python projects. The goal is to remove the tedious and error-prone "update @@ -23,8 +27,8 @@ ## Quick Install * `pip install versioneer` to somewhere to your $PATH -* run `versioneer-installer` in your source tree: this installs `versioneer.py` -* follow the instructions below (also in the `versioneer.py` docstring) +* add a `[versioneer]` section to your setup.cfg (see below) +* run `versioneer install` in your source tree, commit the results ## Version Identifiers @@ -42,7 +46,7 @@ * ask the VCS tool itself, e.g. "git describe" (for checkouts), which knows about recent "tags" and an absolute revision-id * the name of the directory into which the tarball was unpacked -* an expanded VCS variable ($Id$, etc) +* an expanded VCS keyword ($Id$, etc) * a `_version.py` created by some earlier build step For released software, the version identifier is closely related to a VCS @@ -53,7 +57,7 @@ enough information to help developers recreate the same tree, while also giving them an idea of roughly how old the tree is (after version 1.2, before version 1.3). Many VCS systems can report a description that captures this, -for example 'git describe --tags --dirty --always' reports things like +for example `git describe --tags --dirty --always` reports things like "0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the 0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has uncommitted changes. @@ -67,33 +71,50 @@ Versioneer works by adding a special `_version.py` file into your source tree, where your `__init__.py` can import it. This `_version.py` knows how to -dynamically ask the VCS tool for version information at import time. However, -when you use "setup.py build" or "setup.py sdist", `_version.py` in the new -copy is replaced by a small static file that contains just the generated -version data. +dynamically ask the VCS tool for version information at import time. `_version.py` also contains `$Revision$` markers, and the installation process marks `_version.py` to have this marker rewritten with a tag name -during the "git archive" command. As a result, generated tarballs will +during the `git archive` command. As a result, generated tarballs will contain enough information to get the proper version. +To allow `setup.py` to compute a version too, a `versioneer.py` is added to +the top level of your source tree, next to `setup.py` and the `setup.cfg` +that configures it. This overrides several distutils/setuptools commands to +compute the version when invoked, and changes `setup.py build` and `setup.py +sdist` to replace `_version.py` with a small static file that contains just +the generated version data. ## Installation First, decide on values for the following configuration variables: +* `VCS`: the version control system you use. Currently accepts "git". + +* `style`: the style of version string to be produced. See "Styles" below for + details. Defaults to "pep440", which looks like + `TAG[+DISTANCE.gSHORTHASH[.dirty]]`. + * `versionfile_source`: A project-relative pathname into which the generated version strings should be written. This is usually a `_version.py` next to your project's main - `__init__.py` file. If your project uses `src/myproject/__init__.py`, this - should be `src/myproject/_version.py`. This file should be checked in to - your VCS as usual: the copy created below by `setup.py versioneer` will - include code that parses expanded VCS keywords in generated tarballs. The - 'build' and 'sdist' commands will replace it with a copy that has just the - calculated version string. - -* `versionfile_build`: + `__init__.py` file, so it can be imported at runtime. If your project uses + `src/myproject/__init__.py`, this should be `src/myproject/_version.py`. + This file should be checked in to your VCS as usual: the copy created below + by `setup.py setup_versioneer` will include code that parses expanded VCS + keywords in generated tarballs. The 'build' and 'sdist' commands will + replace it with a copy that has just the calculated version string. + + This must be set even if your project does not have any modules (and will + therefore never import `_version.py`), since "setup.py sdist" -based trees + still need somewhere to record the pre-calculated version strings. Anywhere + in the source tree should do. If there is a `__init__.py` next to your + `_version.py`, the `setup.py setup_versioneer` command (described below) + will append some `__version__`-setting assignments, if they aren't already + present. + +* `versionfile_build`: Like `versionfile_source`, but relative to the build directory instead of the source directory. These will differ when your setup.py uses @@ -101,49 +122,69 @@ then you will probably have `versionfile_build='myproject/_version.py'` and `versionfile_source='src/myproject/_version.py'`. + If this is set to None, then `setup.py build` will not attempt to rewrite + any `_version.py` in the built tree. If your project does not have any + libraries (e.g. if it only builds a script), then you should use + `versionfile_build = None` and override `distutils.command.build_scripts` + to explicitly insert a copy of `versioneer.get_version()` into your + generated script. + * `tag_prefix`: a string, like 'PROJECTNAME-', which appears at the start of all VCS tags. If your tags look like 'myproject-1.2.0', then you should use tag_prefix='myproject-'. If you use unprefixed tags like '1.2.0', this - should be an empty string. + should be an empty string, using either `tag_prefix=` or `tag_prefix=''`. * `parentdir_prefix`: - a string, frequently the same as tag_prefix, which appears at the start of - all unpacked tarball filenames. If your tarball unpacks into - 'myproject-1.2.0', this should be 'myproject-'. + a optional string, frequently the same as tag_prefix, which appears at the + start of all unpacked tarball filenames. If your tarball unpacks into + 'myproject-1.2.0', this should be 'myproject-'. To disable this feature, + just omit the field from your `setup.cfg`. -This tool provides one script, named `versioneer-installer`. That script does -one thing: write a copy of `versioneer.py` into the current directory. +This tool provides one script, named `versioneer`. That script has one mode, +"install", which writes a copy of `versioneer.py` into the current directory +and runs `versioneer.py setup` to finish the installation. To versioneer-enable your project: -* 1: Run `versioneer-installer` to copy `versioneer.py` into the top of your - source tree. +* 1: Modify your `setup.cfg`, adding a section named `[versioneer]` and + populating it with the configuration values you decided earlier (note that + the option names are not case-sensitive): + + ```` + [versioneer] + VCS = git + style = pep440 + versionfile_source = src/myproject/_version.py + versionfile_build = myproject/_version.py + tag_prefix = + parentdir_prefix = myproject- + ```` + +* 2: Run `versioneer install`. This will do the following: -* 2: add the following lines to the top of your `setup.py`, with the - configuration values you decided earlier: + * copy `versioneer.py` into the top of your source tree + * create `_version.py` in the right place (`versionfile_source`) + * modify your `__init__.py` (if one exists next to `_version.py`) to define + `__version__` (by calling a function from `_version.py`) + * modify your `MANIFEST.in` to include both `versioneer.py` and the + generated `_version.py` in sdist tarballs - import versioneer - versioneer.versionfile_source = 'src/myproject/_version.py' - versioneer.versionfile_build = 'myproject/_version.py' - versioneer.tag_prefix = '' # tags are like 1.2.0 - versioneer.parentdir_prefix = 'myproject-' # dirname like 'myproject-1.2.0' + `versioneer install` will complain about any problems it finds with your + `setup.py` or `setup.cfg`. Run it multiple times until you have fixed all + the problems. -* 3: add the following arguments to the setup() call in your setup.py: +* 3: add a `import versioneer` to your setup.py, and add the following + arguments to the setup() call: version=versioneer.get_version(), cmdclass=versioneer.get_cmdclass(), -* 4: now run `setup.py versioneer`, which will create `_version.py`, and - will modify your `__init__.py` to define `__version__` (by calling a - function from `_version.py`). It will also modify your `MANIFEST.in` to - include both `versioneer.py` and the generated `_version.py` in sdist - tarballs. - -* 5: commit these changes to your VCS. To make sure you won't forget, - `setup.py versioneer` will mark everything it touched for addition. +* 4: commit these changes to your VCS. To make sure you won't forget, + `versioneer install` will mark everything it touched for addition using + `git add`. Don't forget to add `setup.py` and `setup.cfg` too. ## Post-Installation Usage @@ -163,9 +204,8 @@ * 1: git tag 1.0 * 2: git push; git push --tags -Currently, all version strings must be based upon a tag. Versioneer will -report "unknown" until your tree has at least one tag in its history. This -restriction will be fixed eventually (see issue #12). +Versioneer will report "0+untagged.NUMCOMMITS.gHASH" until your tree has at +least one tag in its history. ## Version-String Flavors @@ -174,52 +214,109 @@ `get_versions()` function. From the "outside" (e.g. in `setup.py`), you can import the top-level `versioneer.py` and run `get_versions()`. -Both functions return a dictionary with different keys for different flavors -of the version string: +Both functions return a dictionary with different flavors of version +information: + +* `['version']`: A condensed version string, rendered using the selected + style. This is the most commonly used value for the project's version + string. The default "pep440" style yields strings like `0.11`, + `0.11+2.g1076c97`, or `0.11+2.g1076c97.dirty`. See the "Styles" section + below for alternative styles. -* `['version']`: condensed tag+distance+shortid+dirty identifier. For git, - this uses the output of `git describe --tags --dirty --always` but strips - the tag_prefix. For example "0.11-2-g1076c97-dirty" indicates that the tree - is like the "1076c97" commit but has uncommitted changes ("-dirty"), and - that this commit is two revisions ("-2-") beyond the "0.11" tag. For - released software (exactly equal to a known tag), the identifier will only - contain the stripped tag, e.g. "0.11". +* `['full-revisionid']`: detailed revision identifier. For Git, this is the + full SHA1 commit id, e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac". -* `['full']`: detailed revision identifier. For Git, this is the full SHA1 - commit id, followed by "-dirty" if the tree contains uncommitted changes, - e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac-dirty". +* `['dirty']`: a boolean, True if the tree has uncommitted changes. Note that + this is only accurate if run in a VCS checkout, otherwise it is likely to + be False or None -Some variants are more useful than others. Including `full` in a bug report -should allow developers to reconstruct the exact code being tested (or -indicate the presence of local changes that should be shared with the +* `['error']`: if the version string could not be computed, this will be set + to a string describing the problem, otherwise it will be None. It may be + useful to throw an exception in setup.py if this is set, to avoid e.g. + creating tarballs with a version string of "unknown". + +Some variants are more useful than others. Including `full-revisionid` in a +bug report should allow developers to reconstruct the exact code being tested +(or indicate the presence of local changes that should be shared with the developers). `version` is suitable for display in an "about" box or a CLI `--version` output: it can be easily compared against release notes and lists of bugs fixed in various releases. -In the future, this will also include a -[PEP-0440](http://legacy.python.org/dev/peps/pep-0440/) -compatible flavor -(e.g. `1.2.post0.dev123`). This loses a lot of information (and has no room -for a hash-based revision id), but is safe to use in a `setup.py` -"`version=`" argument. It also enables tools like *pip* to compare version -strings and evaluate compatibility constraint declarations. - -The `setup.py versioneer` command adds the following text to your -`__init__.py` to place a basic version in `YOURPROJECT.__version__`: +The installer adds the following text to your `__init__.py` to place a basic +version in `YOURPROJECT.__version__`: from ._version import get_versions - __version = get_versions()['version'] + __version__ = get_versions()['version'] del get_versions +## Styles + +The setup.cfg `style=` configuration controls how the VCS information is +rendered into a version string. + +The default style, "pep440", produces a PEP440-compliant string, equal to the +un-prefixed tag name for actual releases, and containing an additional "local +version" section with more detail for in-between builds. For Git, this is +TAG[+DISTANCE.gHEX[.dirty]] , using information from `git describe --tags +--dirty --always`. For example "0.11+2.g1076c97.dirty" indicates that the +tree is like the "1076c97" commit but has uncommitted changes (".dirty"), and +that this commit is two revisions ("+2") beyond the "0.11" tag. For released +software (exactly equal to a known tag), the identifier will only contain the +stripped tag, e.g. "0.11". + +Other styles are available. See details.md in the Versioneer source tree for +descriptions. + +## Debugging + +Versioneer tries to avoid fatal errors: if something goes wrong, it will tend +to return a version of "0+unknown". To investigate the problem, run `setup.py +version`, which will run the version-lookup code in a verbose mode, and will +display the full contents of `get_versions()` (including the `error` string, +which may help identify what went wrong). + ## Updating Versioneer To upgrade your project to a new release of Versioneer, do the following: * install the new Versioneer (`pip install -U versioneer` or equivalent) -* re-run `versioneer-installer` in your source tree to replace `versioneer.py` -* edit `setup.py`, if necessary, to include any new configuration settings indicated by the release notes -* re-run `setup.py versioneer` to replace `SRC/_version.py` +* edit `setup.cfg`, if necessary, to include any new configuration settings + indicated by the release notes +* re-run `versioneer install` in your source tree, to replace + `SRC/_version.py` * commit any changed files +### Upgrading to 0.15 + +Starting with this version, Versioneer is configured with a `[versioneer]` +section in your `setup.cfg` file. Earlier versions required the `setup.py` to +set attributes on the `versioneer` module immediately after import. The new +version will refuse to run (raising an exception during import) until you +have provided the necessary `setup.cfg` section. + +In addition, the Versioneer package provides an executable named +`versioneer`, and the installation process is driven by running `versioneer +install`. In 0.14 and earlier, the executable was named +`versioneer-installer` and was run without an argument. + +### Upgrading to 0.14 + +0.14 changes the format of the version string. 0.13 and earlier used +hyphen-separated strings like "0.11-2-g1076c97-dirty". 0.14 and beyond use a +plus-separated "local version" section strings, with dot-separated +components, like "0.11+2.g1076c97". PEP440-strict tools did not like the old +format, but should be ok with the new one. + +### Upgrading from 0.11 to 0.12 + +Nothing special. + +### Upgrading from 0.10 to 0.11 + +You must add a `versioneer.VCS = "git"` to your `setup.py` before re-running +`setup.py setup_versioneer`. This will enable the use of additional +version-control systems (SVN, etc) in the future. + ## Future Directions This tool is designed to make it easily extended to other version-control @@ -236,52 +333,221 @@ ## License -To make Versioneer easier to embed, all its code is hereby released into the -public domain. The `_version.py` that it creates is also in the public -domain. +To make Versioneer easier to embed, all its code is dedicated to the public +domain. The `_version.py` that it creates is also in the public domain. +Specifically, both are released under the Creative Commons "Public Domain +Dedication" license (CC0-1.0), as described in +https://creativecommons.org/publicdomain/zero/1.0/ . """ -import os, sys, re -from distutils.core import Command -from distutils.command.sdist import sdist as _sdist -from distutils.command.build import build as _build -from distutils.command.build_py import build_py as _build_py +from __future__ import print_function +try: + import configparser +except ImportError: + import ConfigParser as configparser +import errno +import json +import os +import re +import subprocess +import sys + + +class VersioneerConfig: + pass + + +def get_root(): + # we require that all commands are run from the project root, i.e. the + # directory that contains setup.py, setup.cfg, and versioneer.py . + root = os.path.realpath(os.path.abspath(os.getcwd())) + setup_py = os.path.join(root, "setup.py") + versioneer_py = os.path.join(root, "versioneer.py") + if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): + # allow 'python path/to/setup.py COMMAND' + root = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) + setup_py = os.path.join(root, "setup.py") + versioneer_py = os.path.join(root, "versioneer.py") + if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): + err = ("Versioneer was unable to run the project root directory. " + "Versioneer requires setup.py to be executed from " + "its immediate directory (like 'python setup.py COMMAND'), " + "or in a way that lets it use sys.argv[0] to find the root " + "(like 'python path/to/setup.py COMMAND').") + raise VersioneerBadRootError(err) + try: + # Certain runtime workflows (setup.py install/develop in a setuptools + # tree) execute all dependencies in a single python process, so + # "versioneer" may be imported multiple times, and python's shared + # module-import table will cache the first one. So we can't use + # os.path.dirname(__file__), as that will find whichever + # versioneer.py was first imported, even in later projects. + me = os.path.realpath(os.path.abspath(__file__)) + if os.path.splitext(me)[0] != os.path.splitext(versioneer_py)[0]: + print("Warning: build in %s is using versioneer.py from %s" + % (os.path.dirname(me), versioneer_py)) + except NameError: + pass + return root + + +def get_config_from_root(root): + # This might raise EnvironmentError (if setup.cfg is missing), or + # configparser.NoSectionError (if it lacks a [versioneer] section), or + # configparser.NoOptionError (if it lacks "VCS="). See the docstring at + # the top of versioneer.py for instructions on writing your setup.cfg . + setup_cfg = os.path.join(root, "setup.cfg") + parser = configparser.SafeConfigParser() + with open(setup_cfg, "r") as f: + parser.readfp(f) + VCS = parser.get("versioneer", "VCS") # mandatory + + def get(parser, name): + if parser.has_option("versioneer", name): + return parser.get("versioneer", name) + return None + cfg = VersioneerConfig() + cfg.VCS = VCS + cfg.style = get(parser, "style") or "" + cfg.versionfile_source = get(parser, "versionfile_source") + cfg.versionfile_build = get(parser, "versionfile_build") + cfg.tag_prefix = get(parser, "tag_prefix") + if cfg.tag_prefix in ("''", '""'): + cfg.tag_prefix = "" + cfg.parentdir_prefix = get(parser, "parentdir_prefix") + cfg.verbose = get(parser, "verbose") + cfg.lookupfile = get(parser, "lookupfile") + return cfg + + +class NotThisMethod(Exception): + pass -versionfile_source = None -versionfile_build = None -tag_prefix = None -parentdir_prefix = None -lookupfile = None +# these dictionaries contain VCS-specific tools +LONG_VERSION_PY = {} +HANDLERS = {} -VCS = "git" +def register_vcs_handler(vcs, method): # decorator + def decorate(f): + if vcs not in HANDLERS: + HANDLERS[vcs] = {} + HANDLERS[vcs][method] = f + return f + return decorate -LONG_VERSION_PY = ''' + +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): + """Call the given command(s).""" + assert isinstance(commands, list) + p = None + for c in commands: + try: + dispcmd = str([c] + args) + # remember shell=False, so use git.cmd on windows, not just git + p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None)) + break + except EnvironmentError: + e = sys.exc_info()[1] + if e.errno == errno.ENOENT: + continue + if verbose: + print("unable to run %s" % dispcmd) + print(e) + return None + else: + if verbose: + print("unable to find command, tried %s" % (commands,)) + return None + stdout = p.communicate()[0].strip() + if sys.version_info[0] >= 3: + stdout = stdout.decode() + if p.returncode != 0: + if verbose: + print("unable to run %s (error)" % dispcmd) + return None + return stdout +LONG_VERSION_PY['git'] = ''' # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag -# feature). Distribution tarballs (build by setup.py sdist) and build +# feature). Distribution tarballs (built by setup.py sdist) and build # directories (produced by setup.py build) will contain a much shorter file # that just contains the computed version number. # This file is released into the public domain. Generated by -# versioneer-0.10 (https://github.com/warner/python-versioneer) - -# these strings will be replaced by git during git-archive -git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s" -git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s" +# versioneer-0.15+dev (https://github.com/warner/python-versioneer) +"""Git implementation of _version.py.""" +import errno +import os +import re import subprocess import sys -import errno + + +def get_keywords(): + """Get the keywords needed to look up the version information.""" + # these strings will be replaced by git during git-archive. + # setup.py/versioneer.py will grep for the variable names, so they must + # each be defined on a line of their own. _version.py will just call + # get_keywords(). + git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s" + git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s" + keywords = {"refnames": git_refnames, "full": git_full} + return keywords + + +class VersioneerConfig: + + """Container for Versioneer configuration parameters.""" + + +def get_config(): + """Create, populate and return the VersioneerConfig() object.""" + # these strings are filled in when 'setup.py versioneer' creates + # _version.py + cfg = VersioneerConfig() + cfg.VCS = "git" + cfg.style = "%(STYLE)s" + cfg.tag_prefix = "%(TAG_PREFIX)s" + cfg.parentdir_prefix = "%(PARENTDIR_PREFIX)s" + cfg.versionfile_source = "%(VERSIONFILE_SOURCE)s" + cfg.lookupfile = None + cfg.verbose = False + return cfg + + +class NotThisMethod(Exception): + + """Exception raised if a method is not valid for the current scenario.""" + + +LONG_VERSION_PY = {} +HANDLERS = {} + + +def register_vcs_handler(vcs, method): # decorator + """Decorator to mark a method as the handler for a particular VCS.""" + def decorate(f): + """Store f in HANDLERS[vcs][method].""" + if vcs not in HANDLERS: + HANDLERS[vcs] = {} + HANDLERS[vcs][method] = f + return f + return decorate def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): + """Call the given command(s).""" assert isinstance(commands, list) p = None for c in commands: try: + dispcmd = str([c] + args) # remember shell=False, so use git.cmd on windows, not just git p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE, stderr=(subprocess.PIPE if hide_stderr @@ -292,7 +558,7 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): if e.errno == errno.ENOENT: continue if verbose: - print("unable to run %%s" %% args[0]) + print("unable to run %%s" %% dispcmd) print(e) return None else: @@ -300,59 +566,67 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): print("unable to find command, tried %%s" %% (commands,)) return None stdout = p.communicate()[0].strip() - if sys.version >= '3': + if sys.version_info[0] >= 3: stdout = stdout.decode() if p.returncode != 0: if verbose: - print("unable to run %%s (error)" %% args[0]) + print("unable to run %%s (error)" %% dispcmd) return None return stdout -import sys -import re -import os.path +def versions_from_parentdir(parentdir_prefix, root, verbose): + """Try to determine the version from the parent directory name. -def get_gits(root, verbose=False): - if not os.path.exists(os.path.join(root, ".git")): + Source tarballs conventionally unpack into a directory that includes + both the project name and a version string. + """ + dirname = os.path.basename(root) + if not dirname.startswith(parentdir_prefix): if verbose: - print("no .git in %%s" %% root) - return None + print("guessing rootdir is '%%s', but '%%s' doesn't start with " + "prefix '%%s'" %% (root, dirname, parentdir_prefix)) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None} - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - return GITS - -def get_expanded_variables(versionfile_abs): +@register_vcs_handler("git", "get_keywords") +def git_get_keywords(versionfile_abs): + """Extract version information from the given file.""" # the code embedded in _version.py can just fetch the value of these - # variables. When used from setup.py, we don't want to import - # _version.py, so we do it with a regexp instead. This function is not - # used from _version.py. - variables = {} + # keywords. When used from setup.py, we don't want to import _version.py, + # so we do it with a regexp instead. This function is not used from + # _version.py. + keywords = {} try: - f = open(versionfile_abs,"r") + f = open(versionfile_abs, "r") for line in f.readlines(): if line.strip().startswith("git_refnames ="): mo = re.search(r'=\s*"(.*)"', line) if mo: - variables["refnames"] = mo.group(1) + keywords["refnames"] = mo.group(1) if line.strip().startswith("git_full ="): mo = re.search(r'=\s*"(.*)"', line) if mo: - variables["full"] = mo.group(1) + keywords["full"] = mo.group(1) f.close() except EnvironmentError: pass - return variables + return keywords + -def versions_from_expanded_variables(variables, tag_prefix, verbose=False): - refnames = variables["refnames"].strip() +@register_vcs_handler("git", "keywords") +def git_versions_from_keywords(keywords, tag_prefix, verbose): + """Get version information from git keywords.""" + if not keywords: + raise NotThisMethod("no keywords at all, weird") + refnames = keywords["refnames"].strip() if refnames.startswith("$Format"): if verbose: - print("variables are unexpanded, not using") - return {} # unexpanded, so not in an unpacked git-archive tarball + print("keywords are unexpanded, not using") + raise NotThisMethod("unexpanded keywords, not a git-archive tarball") refs = set([r.strip() for r in refnames.strip("()").split(",")]) # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. @@ -371,14 +645,11 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False): print("discarding '%%s', no digits" %% ",".join(refs-tags)) branches = [r for r in refs if not r.startswith(TAG) and r != "HEAD" and not r.startswith("refs/")] + if verbose: + print("likely branches: %%s" %% ",".join(sorted(branches))) + branch = None if branches: branch = branches[0] - else: - branch = "unknown" - - if verbose: - print("likely tags: %s" % ",".join(sorted(tags))) - print("likely branches: %s" % ",".join(sorted(branches))) if verbose: print("likely tags: %%s" %% ",".join(sorted(tags))) @@ -388,250 +659,454 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False): r = ref[len(tag_prefix):] if verbose: print("picking %%s" %% r) - return { "version": r, - "full": variables["full"].strip(), - "branch": branch} - # no suitable tags, so we use the full revision id + + result = {"version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None + } + if branch is not None: + result["branch"] = branch + return result + # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: - print("no suitable tags, using full revision id") - return { "version": variables["full"].strip(), - "full": variables["full"].strip(), - "branch": "unknown"} + print("no suitable tags, using unknown + full revision id") + return {"version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": "no suitable tags"} + -def versions_from_lookup(lookup, root, verbose=False): - GITS = get_gits(root, verbose=verbose) - if GITS is None: - return {} +@register_vcs_handler("git", "pieces_from_vcs") +def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): + """Get version from 'git describe' in the root of the source tree. + + This only gets called if the git-archive 'subst' keywords were *not* + expanded, and _version.py hasn't already been rewritten with a short + version string, meaning we're inside a checked out source tree. + """ + if not os.path.exists(os.path.join(root, ".git")): + if verbose: + print("no .git in %%s" %% root) + raise NotThisMethod("no .git directory") + + GITS = ["git"] + if sys.platform == "win32": + GITS = ["git.cmd", "git.exe"] + # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] + # if there isn't one, this yields HEX[-dirty] (no NUM) + describe_out = run_command(GITS, ["describe", "--tags", "--dirty", + "--always", "--long", + "--match", "%%s*" %% tag_prefix], + cwd=root) + # --long was added in git-1.5.5 + if describe_out is None: + raise NotThisMethod("'git describe' failed") + describe_out = describe_out.strip() + full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + if full_out is None: + raise NotThisMethod("'git rev-parse' failed") + full_out = full_out.strip() + + pieces = {} + pieces["long"] = full_out + pieces["short"] = full_out[:7] # maybe improved later + pieces["error"] = None + + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] + # TAG might have hyphens. + git_describe = describe_out + + # look for -dirty suffix + dirty = git_describe.endswith("-dirty") + pieces["dirty"] = dirty + if dirty: + git_describe = git_describe[:git_describe.rindex("-dirty")] + + # figure out our branch + abbrev_ref_out = run_command(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], cwd=root) + if abbrev_ref_out is not None: + pieces["branch"] = abbrev_ref_out.strip() + + # now we have TAG-NUM-gHEX or HEX + + if "-" in git_describe: + # TAG-NUM-gHEX + mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + if not mo: + # unparseable. Maybe git-describe is misbehaving? + pieces["error"] = ("unable to parse git-describe output: '%%s'" + %% describe_out) + return pieces + + # tag + full_tag = mo.group(1) + if not full_tag.startswith(tag_prefix): + if verbose: + fmt = "tag '%%s' doesn't start with prefix '%%s'" + print(fmt %% (full_tag, tag_prefix)) + pieces["error"] = ("tag '%%s' doesn't start with prefix '%%s'" + %% (full_tag, tag_prefix)) + return pieces + pieces["closest-tag"] = full_tag[len(tag_prefix):] + + # distance: number of commits since tag + pieces["distance"] = int(mo.group(2)) + + # commit: short hex revision ID + pieces["short"] = mo.group(3) + + else: + # HEX: no tags + pieces["closest-tag"] = None + count_out = run_command(GITS, ["rev-list", "HEAD", "--count"], + cwd=root) + pieces["distance"] = int(count_out) # total number of commits + + return pieces + + +@register_vcs_handler("git", "parse_lookup_file") +def git_parse_lookup_file(path): + if not os.path.exists(path): + return [] + + import re + lookup = [] + with open(path, "r") as f: + for line in f: + if '#' in line: + line = line[:line.rindex("#")] + line = line.strip() + try: + split_line = line.split() + if len(split_line) == 3: + pattern, tag, ref_commit = split_line + lookup.append([re.compile(pattern), tag, ref_commit, None]) + elif len(split_line) == 4: + pattern, tag, ref_commit, render = split_line + lookup.append([re.compile(pattern), tag, ref_commit, render]) + elif len(split_line) >= 1: + lookup.append([re.compile(split_line[0]), None, None, None]) + except: + break + return lookup + +@register_vcs_handler("git", "pieces_from_lookup") +def git_pieces_from_lookup(lookup, root, verbose, run_command=run_command): + GITS = ["git"] + if sys.platform == "win32": + GITS = ["git.cmd", "git.exe"] stdout = run_command(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], cwd=root) if stdout is None: - return {} + raise NotThisMethod("git rev-parse --abbrev-ref HEAD failed") current_branch = stdout.strip() - for matcher, tag, ref_commit in lookup: + for matcher, tag, ref_commit, render in lookup: if matcher.match(current_branch): if tag is None or ref_commit is None: - return {} + raise NotThisMethod("tag or ref_commit is unset for this branch") stdout = run_command(GITS, ["rev-list", "%%s..HEAD" %% ref_commit, "--count"], cwd=root) if stdout is None: - return {} - num_commits = stdout.strip() + raise NotThisMethod("git rev-list %%s..HEAD --count failed" %% ref_commit) + try: + num_commits = int(stdout.strip()) + except ValueError: + raise NotThisMethod("git rev-list %%s..HEAD --count didn't return a valid number" %% ref_commit) stdout =run_command(GITS, ["rev-parse", "--short", "HEAD"], cwd=root) if stdout is None: - return {} + raise NotThisMethod("git describe rev-parse --short HEAD failed") short_hash = stdout.strip() stdout = run_command(GITS, ["describe", "--tags", "--dirty", "--always"], cwd=root) if stdout is None: - return {} + raise NotThisMethod("git describe --tags --dirty --always failed") dirty = stdout.strip().endswith("-dirty") stdout = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) if stdout is None: - return {} + raise NotThisMethod("git rev-parse HEAD failed") full = stdout.strip() - version = "%%s-%%s-g%%s" %% (tag, num_commits, short_hash) - if dirty: - version += "-dirty" - full += "-dirty" - return {"version": version, "full": full, "branch": current_branch} + return { + "long": full, + "short": short_hash, + "dirty": dirty, + "branch": current_branch, + "closest-tag": tag, + "distance": num_commits, + "error": None, + "render": render + } - return {} + raise NotThisMethod("no matching lookup definition found") -def versions_from_vcs(tag_prefix, root, verbose=False): - # this runs 'git' from the root of the source tree. This only gets called - # if the git-archive 'subst' variables were *not* expanded, and - # _version.py hasn't already been rewritten with a short version string, - # meaning we're inside a checked out source tree. - GITS = get_gits(root, verbose=verbose) - if GITS is None: - return {} +def plus_or_dot(pieces): + """Return a + if we don't already have one, else return a .""" + if "+" in pieces.get("closest-tag", ""): + return "." + return "+" - stdout = run_command(GITS, ["describe", "--tags", "--dirty", "--always"], - cwd=root) - if stdout is None: - return {} - if not stdout.startswith(tag_prefix): - if verbose: - print("tag '%%s' doesn't start with prefix '%%s'" %% (stdout, tag_prefix)) - return {} - tag = stdout[len(tag_prefix):] - stdout = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) - if stdout is None: - return {} - full = stdout.strip() - if tag.endswith("-dirty"): - full += "-dirty" - stdout = run_command(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], - cwd=root) - if stdout is None: - branch = None +def render_pep440(pieces): + """Build up version string, with post-release "local version identifier". + + Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you + get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + + Exceptions: + 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += plus_or_dot(pieces) + rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" else: - branch = stdout.strip() - return {"version": tag, "full": full, "branch": branch} + # exception #1 + rendered = "0+untagged.%%d.g%%s" %% (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_pre(pieces): + """TAG[.post.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post.devDISTANCE + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += ".post.dev%%d" %% pieces["distance"] + else: + # exception #1 + rendered = "0.post.dev%%d" %% pieces["distance"] + return rendered + + +def render_pep440_post(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX] . + + The ".dev0" means dirty. Note that .dev0 sorts backwards + (a dirty tree will appear "older" than the corresponding clean one), + but you shouldn't be releasing software with -dirty anyways. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%%s" %% pieces["short"] + else: + # exception #1 + rendered = "0.post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += "+g%%s" %% pieces["short"] + return rendered + + +def render_pep440_old(pieces): + """TAG[.postDISTANCE[.dev0]] . + + The ".dev0" means dirty. + + Eexceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + else: + # exception #1 + rendered = "0.post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + return rendered -def versions_from_parentdir(parentdir_prefix, root, verbose=False): - # Source tarballs conventionally unpack into a directory that includes - # both the project name and a version string. - dirname = os.path.basename(root) - if not dirname.startswith(parentdir_prefix): - if verbose: - print("guessing rootdir is '%%s', but '%%s' doesn't start with prefix '%%s'" %% - (root, dirname, parentdir_prefix)) - return None - return {"version": dirname[len(parentdir_prefix):], "full": "", "branch": ""} +def render_git_describe(pieces): + """TAG[-DISTANCE-gHEX][-dirty]. -tag_prefix = "%(TAG_PREFIX)s" -parentdir_prefix = "%(PARENTDIR_PREFIX)s" -versionfile_source = "%(VERSIONFILE_SOURCE)s" -lookupfile = %(LOOKUPFILE)s + Like 'git describe --tags --dirty --always'. -def parse_lookup_file(root, lookup_path=None): - if not lookup_path: - lookup_path = lookupfile - if not lookup_path: - return [] + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered - path = os.path.join(root, lookup_path) - if not os.path.exists(path): - return [] - import re - lookup = [] - with open(os.path.join(root, lookup_path), "r") as f: - for line in f: - if '#' in line: - line = line[:line.rindex('#')] - line = line.strip() - try: - split_line = line.split() - if len(split_line) == 3: - pattern, tag, ref_commit = split_line - lookup.append([re.compile(pattern), tag, ref_commit]) - elif len(split_line) >= 1: - lookup.append([re.compile(split_line[0]), None, None]) - except: - break - return lookup +def render_git_describe_long(pieces): + """TAG-DISTANCE-gHEX[-dirty]. -def get_versions(default={"version": "unknown", "full": "", "branch": "unknown"}, lookup_path=None, verbose=False): + Like 'git describe --tags --dirty --always -long'. + The distance/hash is unconditional. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render(pieces, style): + """Render the given version pieces into the requested style.""" + if pieces["error"]: + return {"version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"]} + + if "render" in pieces and pieces["render"] is not None: + style = pieces["render"] + + if not style or style == "default": + style = "pep440" # the default + + if style == "pep440": + rendered = render_pep440(pieces) + elif style == "pep440-pre": + rendered = render_pep440_pre(pieces) + elif style == "pep440-post": + rendered = render_pep440_post(pieces) + elif style == "pep440-old": + rendered = render_pep440_old(pieces) + elif style == "git-describe": + rendered = render_git_describe(pieces) + elif style == "git-describe-long": + rendered = render_git_describe_long(pieces) + else: + raise ValueError("unknown style '%%s'" %% style) + + result = {"version": rendered, "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], "error": None} + if "branch" in pieces and pieces["branch"] is not None: + result["branch"] = pieces["branch"] + return result + + +def get_versions(): + """Get version information or return default if unable to do so.""" # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have # __file__, we can work backwards from there to the root. Some # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which - # case we can only use expanded variables. + # case we can only use expanded keywords. - variables = { "refnames": git_refnames, "full": git_full } - ver = versions_from_expanded_variables(variables, tag_prefix, verbose) - if ver: - return ver + cfg = get_config() + verbose = cfg.verbose try: - root = os.path.abspath(__file__) + return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, + verbose) + except NotThisMethod: + pass + + try: + root = os.path.realpath(__file__) # versionfile_source is the relative path from the top of the source # tree (where the .git directory might live) to this file. Invert # this to find the root from __file__. - for i in range(len(versionfile_source.split("/"))): + for i in cfg.versionfile_source.split('/'): root = os.path.dirname(root) except NameError: - return default - - lookup = parse_lookup_file(root, lookup_path=lookup_path) - return (versions_from_lookup(lookup, root, verbose) - or versions_from_vcs(tag_prefix, root, verbose) - or versions_from_parentdir(parentdir_prefix, root, verbose) - or default) - -''' + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to find root of source tree"} - -import subprocess -import sys -import errno - - -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): - assert isinstance(commands, list) - p = None - for c in commands: + lookupfile = os.path.join(root, cfg.lookupfile if cfg.lookupfile is not None else ".versioneer-lookup") + if os.path.exists(lookupfile): try: - # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) - break - except EnvironmentError: - e = sys.exc_info()[1] - if e.errno == errno.ENOENT: - continue - if verbose: - print("unable to run %s" % args[0]) - print(e) - return None - else: - if verbose: - print("unable to find command, tried %s" % (commands,)) - return None - stdout = p.communicate()[0].strip() - if sys.version >= '3': - stdout = stdout.decode() - if p.returncode != 0: - if verbose: - print("unable to run %s (error)" % args[0]) - return None - return stdout - + lookup_data = git_parse_lookup_file(lookupfile) + pieces = git_pieces_from_lookup(lookup_data, root, verbose) + return render(pieces, cfg.style) + except NotThisMethod: + pass -import sys -import re -import os.path + try: + pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) + return render(pieces, cfg.style) + except NotThisMethod: + pass -def get_gits(root, verbose=False): - if not os.path.exists(os.path.join(root, ".git")): - if verbose: - print("no .git in %s" % root) - return None + try: + if cfg.parentdir_prefix: + return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) + except NotThisMethod: + pass - GITS = ["git"] - if sys.platform == "win32": - GITS = ["git.cmd", "git.exe"] - return GITS + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to compute version"} +''' -def get_expanded_variables(versionfile_abs): +@register_vcs_handler("git", "get_keywords") +def git_get_keywords(versionfile_abs): + """Extract version information from the given file.""" # the code embedded in _version.py can just fetch the value of these - # variables. When used from setup.py, we don't want to import - # _version.py, so we do it with a regexp instead. This function is not - # used from _version.py. - variables = {} + # keywords. When used from setup.py, we don't want to import _version.py, + # so we do it with a regexp instead. This function is not used from + # _version.py. + keywords = {} try: - f = open(versionfile_abs,"r") + f = open(versionfile_abs, "r") for line in f.readlines(): if line.strip().startswith("git_refnames ="): mo = re.search(r'=\s*"(.*)"', line) if mo: - variables["refnames"] = mo.group(1) + keywords["refnames"] = mo.group(1) if line.strip().startswith("git_full ="): mo = re.search(r'=\s*"(.*)"', line) if mo: - variables["full"] = mo.group(1) + keywords["full"] = mo.group(1) f.close() except EnvironmentError: pass - return variables + return keywords + -def versions_from_expanded_variables(variables, tag_prefix, verbose=False): - refnames = variables["refnames"].strip() +@register_vcs_handler("git", "keywords") +def git_versions_from_keywords(keywords, tag_prefix, verbose): + """Get version information from git keywords.""" + if not keywords: + raise NotThisMethod("no keywords at all, weird") + refnames = keywords["refnames"].strip() if refnames.startswith("$Format"): if verbose: - print("variables are unexpanded, not using") - return {} # unexpanded, so not in an unpacked git-archive tarball + print("keywords are unexpanded, not using") + raise NotThisMethod("unexpanded keywords, not a git-archive tarball") refs = set([r.strip() for r in refnames.strip("()").split(",")]) # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. @@ -650,153 +1125,217 @@ def versions_from_expanded_variables(variables, tag_prefix, verbose=False): print("discarding '%s', no digits" % ",".join(refs-tags)) branches = [r for r in refs if not r.startswith(TAG) and r != "HEAD" and not r.startswith("refs/")] + if verbose: + print("likely branches: %s" % ",".join(sorted(branches))) + branch = None if branches: branch = branches[0] - else: - branch = "unknown" if verbose: print("likely tags: %s" % ",".join(sorted(tags))) - print("likely branches: %s" % ",".join(sorted(branches))) - for ref in sorted(tags): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): r = ref[len(tag_prefix):] if verbose: print("picking %s" % r) - return { "version": r, - "full": variables["full"].strip(), - "branch": branch } - # no suitable tags, so we use the full revision id + + result = {"version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None + } + if branch is not None: + result["branch"] = branch + return result + # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: - print("no suitable tags, using full revision id") - return { "version": variables["full"].strip(), - "full": variables["full"].strip(), - "branch": "unknown"} + print("no suitable tags, using unknown + full revision id") + return {"version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": "no suitable tags"} + + +@register_vcs_handler("git", "pieces_from_vcs") +def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): + """Get version from 'git describe' in the root of the source tree. + + This only gets called if the git-archive 'subst' keywords were *not* + expanded, and _version.py hasn't already been rewritten with a short + version string, meaning we're inside a checked out source tree. + """ + if not os.path.exists(os.path.join(root, ".git")): + if verbose: + print("no .git in %s" % root) + raise NotThisMethod("no .git directory") + + GITS = ["git"] + if sys.platform == "win32": + GITS = ["git.cmd", "git.exe"] + # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] + # if there isn't one, this yields HEX[-dirty] (no NUM) + describe_out = run_command(GITS, ["describe", "--tags", "--dirty", + "--always", "--long", + "--match", "%s*" % tag_prefix], + cwd=root) + # --long was added in git-1.5.5 + if describe_out is None: + raise NotThisMethod("'git describe' failed") + describe_out = describe_out.strip() + full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + if full_out is None: + raise NotThisMethod("'git rev-parse' failed") + full_out = full_out.strip() + + pieces = {} + pieces["long"] = full_out + pieces["short"] = full_out[:7] # maybe improved later + pieces["error"] = None + + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] + # TAG might have hyphens. + git_describe = describe_out + + # look for -dirty suffix + dirty = git_describe.endswith("-dirty") + pieces["dirty"] = dirty + if dirty: + git_describe = git_describe[:git_describe.rindex("-dirty")] + + # figure out our branch + abbrev_ref_out = run_command(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], cwd=root) + if abbrev_ref_out is not None: + pieces["branch"] = abbrev_ref_out.strip() + + # now we have TAG-NUM-gHEX or HEX + + if "-" in git_describe: + # TAG-NUM-gHEX + mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + if not mo: + # unparseable. Maybe git-describe is misbehaving? + pieces["error"] = ("unable to parse git-describe output: '%s'" + % describe_out) + return pieces + + # tag + full_tag = mo.group(1) + if not full_tag.startswith(tag_prefix): + if verbose: + fmt = "tag '%s' doesn't start with prefix '%s'" + print(fmt % (full_tag, tag_prefix)) + pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" + % (full_tag, tag_prefix)) + return pieces + pieces["closest-tag"] = full_tag[len(tag_prefix):] + + # distance: number of commits since tag + pieces["distance"] = int(mo.group(2)) + + # commit: short hex revision ID + pieces["short"] = mo.group(3) + + else: + # HEX: no tags + pieces["closest-tag"] = None + count_out = run_command(GITS, ["rev-list", "HEAD", "--count"], + cwd=root) + pieces["distance"] = int(count_out) # total number of commits + + return pieces + + +@register_vcs_handler("git", "parse_lookup_file") +def git_parse_lookup_file(path): + if not os.path.exists(path): + return [] + import re + lookup = [] + with open(path, "r") as f: + for line in f: + if '#' in line: + line = line[:line.rindex("#")] + line = line.strip() + try: + split_line = line.split() + if len(split_line) == 3: + pattern, tag, ref_commit = split_line + lookup.append([re.compile(pattern), tag, ref_commit, None]) + elif len(split_line) == 4: + pattern, tag, ref_commit, render = split_line + lookup.append([re.compile(pattern), tag, ref_commit, render]) + elif len(split_line) >= 1: + lookup.append([re.compile(split_line[0]), None, None, None]) + except: + break + return lookup -def versions_from_lookup(lookup, root, verbose=False): - GITS = get_gits(root, verbose=verbose) - if GITS is None: - return {} +@register_vcs_handler("git", "pieces_from_lookup") +def git_pieces_from_lookup(lookup, root, verbose, run_command=run_command): + GITS = ["git"] + if sys.platform == "win32": + GITS = ["git.cmd", "git.exe"] stdout = run_command(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], cwd=root) if stdout is None: - return {} + raise NotThisMethod("git rev-parse --abbrev-ref HEAD failed") current_branch = stdout.strip() - for matcher, tag, ref_commit in lookup: + for matcher, tag, ref_commit, render in lookup: if matcher.match(current_branch): if tag is None or ref_commit is None: - return {} + raise NotThisMethod("tag or ref_commit is unset for this branch") stdout = run_command(GITS, ["rev-list", "%s..HEAD" % ref_commit, "--count"], cwd=root) if stdout is None: - return {} - num_commits = stdout.strip() + raise NotThisMethod("git rev-list %s..HEAD --count failed" % ref_commit) + try: + num_commits = int(stdout.strip()) + except ValueError: + raise NotThisMethod("git rev-list %s..HEAD --count didn't return a valid number" % ref_commit) stdout =run_command(GITS, ["rev-parse", "--short", "HEAD"], cwd=root) if stdout is None: - return {} + raise NotThisMethod("git describe rev-parse --short HEAD failed") short_hash = stdout.strip() stdout = run_command(GITS, ["describe", "--tags", "--dirty", "--always"], cwd=root) if stdout is None: - return {} + raise NotThisMethod("git describe --tags --dirty --always failed") dirty = stdout.strip().endswith("-dirty") stdout = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) if stdout is None: - return {} + raise NotThisMethod("git rev-parse HEAD failed") full = stdout.strip() - version = "%s-%s-g%s" % (tag, num_commits, short_hash) - if dirty: - version += "-dirty" - full += "-dirty" - return {"version": version, "full": full, "branch": current_branch} - - return {} - - -def versions_from_vcs(tag_prefix, root, verbose=False): - # this runs 'git' from the root of the source tree. This only gets called - # if the git-archive 'subst' variables were *not* expanded, and - # _version.py hasn't already been rewritten with a short version string, - # meaning we're inside a checked out source tree. - - GITS = get_gits(root, verbose=verbose) - if GITS is None: - return {} - - stdout = run_command(GITS, ["describe", "--tags", "--dirty", "--always"], - cwd=root) - if stdout is None: - return {} - if not stdout.startswith(tag_prefix): - if verbose: - print("tag '%s' doesn't start with prefix '%s'" % (stdout, tag_prefix)) - return {} - tag = stdout[len(tag_prefix):] - stdout = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) - if stdout is None: - return {} - full = stdout.strip() - if tag.endswith("-dirty"): - full += "-dirty" - - stdout = run_command(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], - cwd=root) - if stdout is None: - branch = None - else: - branch = stdout.strip() - return {"version": tag, "full": full, "branch": branch} - - -def versions_from_parentdir(parentdir_prefix, root, verbose=False): - # Source tarballs conventionally unpack into a directory that includes - # both the project name and a version string. - dirname = os.path.basename(root) - if not dirname.startswith(parentdir_prefix): - if verbose: - print("guessing rootdir is '%s', but '%s' doesn't start with prefix '%s'" % - (root, dirname, parentdir_prefix)) - return None - return {"version": dirname[len(parentdir_prefix):], "full": "", "branch": ""} -import os.path -import sys - -# os.path.relpath only appeared in Python-2.6 . Define it here for 2.5. -def os_path_relpath(path, start=os.path.curdir): - """Return a relative version of a path""" - - if not path: - raise ValueError("no path specified") - - start_list = [x for x in os.path.abspath(start).split(os.path.sep) if x] - path_list = [x for x in os.path.abspath(path).split(os.path.sep) if x] + return { + "long": full, + "short": short_hash, + "dirty": dirty, + "branch": current_branch, + "closest-tag": tag, + "distance": num_commits, + "error": None, + "render": render + } - # Work out how much of the filepath is shared by start and path. - i = len(os.path.commonprefix([start_list, path_list])) + raise NotThisMethod("no matching lookup definition found") - rel_list = [os.path.pardir] * (len(start_list)-i) + path_list[i:] - if not rel_list: - return os.path.curdir - return os.path.join(*rel_list) def do_vcs_install(manifest_in, versionfile_source, ipy): GITS = ["git"] if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] - files = [manifest_in, versionfile_source, ipy] + files = [manifest_in, versionfile_source] + if ipy: + files.append(ipy) try: me = __file__ if me.endswith(".pyc") or me.endswith(".pyo"): me = os.path.splitext(me)[0] + ".py" - versioneer_file = os_path_relpath(me) + versioneer_file = os.path.relpath(me) except NameError: versioneer_file = "versioneer.py" files.append(versioneer_file) @@ -809,7 +1348,7 @@ def do_vcs_install(manifest_in, versionfile_source, ipy): present = True f.close() except EnvironmentError: - pass + pass if not present: f = open(".gitattributes", "a+") f.write("%s export-subst\n" % versionfile_source) @@ -817,221 +1356,498 @@ def do_vcs_install(manifest_in, versionfile_source, ipy): files.append(".gitattributes") run_command(GITS, ["add", "--"] + files) + +def versions_from_parentdir(parentdir_prefix, root, verbose): + """Try to determine the version from the parent directory name. + + Source tarballs conventionally unpack into a directory that includes + both the project name and a version string. + """ + dirname = os.path.basename(root) + if not dirname.startswith(parentdir_prefix): + if verbose: + print("guessing rootdir is '%s', but '%s' doesn't start with " + "prefix '%s'" % (root, dirname, parentdir_prefix)) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None} + SHORT_VERSION_PY = """ -# This file was generated by 'versioneer.py' (0.10) from +# This file was generated by 'versioneer.py' (0.15+dev) from # revision-control system data, or from the parent directory name of an # unpacked source archive. Distribution tarballs contain a pre-generated copy # of this file. -version_version = '%(version)s' -version_full = '%(full)s' -version_branch = '%(branch)s' -def get_versions(default={}, verbose=False): - return {'version': version_version, 'full': version_full, 'branch': version_branch} +import json +import sys + +version_json = ''' +%s +''' # END VERSION_JSON + +def get_versions(): + return json.loads(version_json) """ -DEFAULT = {"version": "unknown", "full": "unknown", "branch": "unknown"} def versions_from_file(filename): - versions = {} try: - f = open(filename) + with open(filename) as f: + contents = f.read() except EnvironmentError: - return versions - for line in f.readlines(): - mo = re.match("version_version = '([^']+)'", line) - if mo: - versions["version"] = mo.group(1) - mo = re.match("version_full = '([^']+)'", line) - if mo: - versions["full"] = mo.group(1) - mo = re.match("version_branch = '([^']+)'", line) - if mo: - versions["branch"] = mo.group(1) - f.close() - return versions + raise NotThisMethod("unable to read _version.py") + mo = re.search(r"version_json = '''\n(.*)''' # END VERSION_JSON", + contents, re.M | re.S) + if not mo: + raise NotThisMethod("no version_json in _version.py") + return json.loads(mo.group(1)) + def write_to_version_file(filename, versions): - f = open(filename, "w") - f.write(SHORT_VERSION_PY % versions) - f.close() + os.unlink(filename) + contents = json.dumps(versions, sort_keys=True, + indent=1, separators=(",", ": ")) + with open(filename, "w") as f: + f.write(SHORT_VERSION_PY % contents) + print("set %s to '%s'" % (filename, versions["version"])) -def get_root(): - try: - return os.path.dirname(os.path.abspath(__file__)) - except NameError: - return os.path.dirname(os.path.abspath(sys.argv[0])) -def parse_lookup_file(root, lookup_path=None): - if not lookup_path: - lookup_path = lookupfile +def plus_or_dot(pieces): + """Return a + if we don't already have one, else return a .""" + if "+" in pieces.get("closest-tag", ""): + return "." + return "+" - path = os.path.join(root, lookup_path) - if not os.path.exists(path): - return [] - import re - lookup = [] - with open(os.path.join(root, lookup_path), "r") as f: - for line in f: - if '#' in line: - line = line[:line.rindex("#")] - line = line.strip() - try: - split_line = line.split() - if len(split_line) == 3: - pattern, tag, ref_commit = split_line - lookup.append([re.compile(pattern), tag, ref_commit]) - elif len(split_line) >= 1: - lookup.append([re.compile(split_line[0]), None, None]) - except: - break - return lookup +def render_pep440(pieces): + """Build up version string, with post-release "local version identifier". + + Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you + get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + + Exceptions: + 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += plus_or_dot(pieces) + rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0+untagged.%d.g%s" % (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_pre(pieces): + """TAG[.post.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post.devDISTANCE + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += ".post.dev%d" % pieces["distance"] + else: + # exception #1 + rendered = "0.post.dev%d" % pieces["distance"] + return rendered + + +def render_pep440_post(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX] . + + The ".dev0" means dirty. Note that .dev0 sorts backwards + (a dirty tree will appear "older" than the corresponding clean one), + but you shouldn't be releasing software with -dirty anyways. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%s" % pieces["short"] + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += "+g%s" % pieces["short"] + return rendered + + +def render_pep440_old(pieces): + """TAG[.postDISTANCE[.dev0]] . + + The ".dev0" means dirty. + + Eexceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + return rendered -def get_versions(default=DEFAULT, verbose=False): - # returns dict with three keys: 'version', 'full' and 'branch' - assert versionfile_source is not None, "please set versioneer.versionfile_source" - assert tag_prefix is not None, "please set versioneer.tag_prefix" - assert parentdir_prefix is not None, "please set versioneer.parentdir_prefix" - - # I am in versioneer.py, which must live at the top of the source tree, - # which we use to compute the root directory. py2exe/bbfreeze/non-CPython - # don't have __file__, in which case we fall back to sys.argv[0] (which - # ought to be the setup.py script). We prefer __file__ since that's more - # robust in cases where setup.py was invoked in some weird way (e.g. pip) - root = get_root() - versionfile_abs = os.path.join(root, versionfile_source) - if lookupfile: - lookup = parse_lookup_file(root, lookup_path = lookupfile) +def render_git_describe(pieces): + """TAG[-DISTANCE-gHEX][-dirty]. + + Like 'git describe --tags --dirty --always'. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render_git_describe_long(pieces): + """TAG-DISTANCE-gHEX[-dirty]. + + Like 'git describe --tags --dirty --always -long'. + The distance/hash is unconditional. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) else: - lookup = None - - # extract version from first of _version.py, 'git describe', parentdir. - # This is meant to work for developers using a source checkout, for users - # of a tarball created by 'setup.py sdist', and for users of a - # tarball/zipball created by 'git archive' or github's download-from-tag - # feature. - - variables = get_expanded_variables(versionfile_abs) - if variables: - ver = versions_from_expanded_variables(variables, tag_prefix) - if ver: - if verbose: print("got version from expanded variable %s" % ver) + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render(pieces, style): + """Render the given version pieces into the requested style.""" + if pieces["error"]: + return {"version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"]} + + if "render" in pieces and pieces["render"] is not None: + style = pieces["render"] + + if not style or style == "default": + style = "pep440" # the default + + if style == "pep440": + rendered = render_pep440(pieces) + elif style == "pep440-pre": + rendered = render_pep440_pre(pieces) + elif style == "pep440-post": + rendered = render_pep440_post(pieces) + elif style == "pep440-old": + rendered = render_pep440_old(pieces) + elif style == "git-describe": + rendered = render_git_describe(pieces) + elif style == "git-describe-long": + rendered = render_git_describe_long(pieces) + else: + raise ValueError("unknown style '%s'" % style) + + result = {"version": rendered, "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], "error": None} + if "branch" in pieces and pieces["branch"] is not None: + result["branch"] = pieces["branch"] + return result + + +class VersioneerBadRootError(Exception): + pass + + +def get_versions(verbose=False): + # returns dict with two keys: 'version' and 'full' + + if "versioneer" in sys.modules: + # see the discussion in cmdclass.py:get_cmdclass() + del sys.modules["versioneer"] + + root = get_root() + cfg = get_config_from_root(root) + + assert cfg.VCS is not None, "please set [versioneer]VCS= in setup.cfg" + handlers = HANDLERS.get(cfg.VCS) + assert handlers, "unrecognized VCS '%s'" % cfg.VCS + verbose = verbose or cfg.verbose + assert cfg.versionfile_source is not None, \ + "please set versioneer.versionfile_source" + assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" + + versionfile_abs = os.path.join(root, cfg.versionfile_source) + + # extract version from first of: _version.py, VCS command (e.g. 'git + # describe'), parentdir. This is meant to work for developers using a + # source checkout, for users of a tarball created by 'setup.py sdist', + # and for users of a tarball/zipball created by 'git archive' or github's + # download-from-tag feature or the equivalent in other VCSes. + + get_keywords_f = handlers.get("get_keywords") + from_keywords_f = handlers.get("keywords") + if get_keywords_f and from_keywords_f: + try: + keywords = get_keywords_f(versionfile_abs) + ver = from_keywords_f(keywords, cfg.tag_prefix, verbose) + if verbose: + print("got version from expanded keyword %s" % ver) return ver + except NotThisMethod: + pass - ver = versions_from_file(versionfile_abs) - if ver: - if verbose: print("got version from file %s %s" % (versionfile_abs,ver)) + try: + ver = versions_from_file(versionfile_abs) + if verbose: + print("got version from file %s %s" % (versionfile_abs, ver)) return ver + except NotThisMethod: + pass - if lookup: - ver = versions_from_lookup(lookup, root, verbose=verbose) - if ver: - if verbose: print("got version from lookup %s" % ver) + lookupfile = os.path.join(root, cfg.lookupfile if cfg.lookupfile is not None else ".versioneer-lookup") + if os.path.exists(lookupfile): + parse_lookup_file_f = handlers.get("parse_lookup_file") + versions_from_lookup_f = handlers.get("pieces_from_lookup") + if parse_lookup_file_f and versions_from_lookup_f: + try: + lookup_data = parse_lookup_file_f(lookupfile) + pieces = versions_from_lookup_f(lookup_data, root, verbose) + ver = render(pieces, cfg.style) + if verbose: + print("got version from lookup file") + return ver + except NotThisMethod: + pass + elif verbose: + print("lookup file %s doesn't exist") + + from_vcs_f = handlers.get("pieces_from_vcs") + if from_vcs_f: + try: + pieces = from_vcs_f(cfg.tag_prefix, root, verbose) + ver = render(pieces, cfg.style) + if verbose: + print("got version from VCS %s" % ver) return ver + except NotThisMethod: + pass - ver = versions_from_vcs(tag_prefix, root, verbose) - if ver: - if verbose: print("got version from git %s" % ver) - return ver + try: + if cfg.parentdir_prefix: + ver = versions_from_parentdir(cfg.parentdir_prefix, root, verbose) + if verbose: + print("got version from parentdir %s" % ver) + return ver + except NotThisMethod: + pass - ver = versions_from_parentdir(parentdir_prefix, root, verbose) - if ver: - if verbose: print("got version from parentdir %s" % ver) - return ver + if verbose: + print("unable to compute version") - if verbose: print("got version from default %s" % ver) - return default + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, "error": "unable to compute version"} -def get_version(verbose=False): - return get_versions(verbose=verbose)["version"] -class cmd_version(Command): - description = "report generated version string" - user_options = [] - boolean_options = [] - def initialize_options(self): - pass - def finalize_options(self): - pass - def run(self): - ver = get_version(verbose=True) - print("Version is currently: %s" % ver) - -class cmd_build_py(_build_py): - def run(self): - _build_py.run(self) - - versions = get_versions(verbose=True) - # now locate _version.py in the new build/ directory and replace it - # with an updated value - target_versionfile = os.path.join(self.build_lib, versionfile_build) - print("UPDATING %s" % target_versionfile) - if os.path.exists(target_versionfile): - os.unlink(target_versionfile) - f = open(target_versionfile, "w") - f.write(SHORT_VERSION_PY % versions) - f.close() +def get_version(): + return get_versions()["version"] -class cmd_build(_build): - def run(self): - _build.run(self) - - versions = get_versions(verbose=True) - # now locate _version.py in the new build/ directory and replace it - # with an updated value - target_versionfile = os.path.join(self.build_lib, versionfile_build) - print("UPDATING %s" % target_versionfile) - if os.path.exists(target_versionfile): - os.unlink(target_versionfile) - f = open(target_versionfile, "w") - f.write(SHORT_VERSION_PY % versions) - f.close() -if 'cx_Freeze' in sys.modules: # cx_freeze enabled? - from cx_Freeze.dist import build_exe as _build_exe +def get_cmdclass(): + if "versioneer" in sys.modules: + del sys.modules["versioneer"] + # this fixes the "python setup.py develop" case (also 'install' and + # 'easy_install .'), in which subdependencies of the main project are + # built (using setup.py bdist_egg) in the same python process. Assume + # a main project A and a dependency B, which use different versions + # of Versioneer. A's setup.py imports A's Versioneer, leaving it in + # sys.modules by the time B's setup.py is executed, causing B to run + # with the wrong versioneer. Setuptools wraps the sub-dep builds in a + # sandbox that restores sys.modules to it's pre-build state, so the + # parent is protected against the child's "import versioneer". By + # removing ourselves from sys.modules here, before the child build + # happens, we protect the child from the parent's versioneer too. + # Also see https://github.com/warner/python-versioneer/issues/52 + + cmds = {} + + # we add "version" to both distutils and setuptools + from distutils.core import Command + + class cmd_version(Command): + description = "report generated version string" + user_options = [] + boolean_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass - class cmd_build_exe(_build_exe): def run(self): - versions = get_versions(verbose=True) - target_versionfile = versionfile_source + vers = get_versions(verbose=True) + print("Version: %s" % vers["version"]) + print(" full-revisionid: %s" % vers.get("full-revisionid")) + print(" dirty: %s" % vers.get("dirty")) + if vers["error"]: + print(" error: %s" % vers["error"]) + cmds["version"] = cmd_version + + # we override "build_py" in both distutils and setuptools + # + # most invocation pathways end up running build_py: + # distutils/build -> build_py + # distutils/install -> distutils/build ->.. + # setuptools/bdist_wheel -> distutils/install ->.. + # setuptools/bdist_egg -> distutils/install_lib -> build_py + # setuptools/install -> bdist_egg ->.. + # setuptools/develop -> ? + + from distutils.command.build_py import build_py as _build_py + + class cmd_build_py(_build_py): + def run(self): + root = get_root() + cfg = get_config_from_root(root) + versions = get_versions() + _build_py.run(self) + # now locate _version.py in the new build/ directory and replace + # it with an updated value + if cfg.versionfile_build: + target_versionfile = os.path.join(self.build_lib, + cfg.versionfile_build) + print("UPDATING %s" % target_versionfile) + write_to_version_file(target_versionfile, versions) + cmds["build_py"] = cmd_build_py + + if "cx_Freeze" in sys.modules: # cx_freeze enabled? + from cx_Freeze.dist import build_exe as _build_exe + + class cmd_build_exe(_build_exe): + def run(self): + root = get_root() + cfg = get_config_from_root(root) + versions = get_versions() + target_versionfile = cfg.versionfile_source + print("UPDATING %s" % target_versionfile) + write_to_version_file(target_versionfile, versions) + + _build_exe.run(self) + os.unlink(target_versionfile) + with open(cfg.versionfile_source, "w") as f: + LONG = LONG_VERSION_PY[cfg.VCS] + f.write(LONG % + {"DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + }) + cmds["build_exe"] = cmd_build_exe + del cmds["build_py"] + + # we override different "sdist" commands for both environments + if "setuptools" in sys.modules: + from setuptools.command.sdist import sdist as _sdist + else: + from distutils.command.sdist import sdist as _sdist + + class cmd_sdist(_sdist): + def run(self): + versions = get_versions() + self._versioneer_generated_versions = versions + # unless we update this, the command will keep using the old + # version + self.distribution.metadata.version = versions["version"] + return _sdist.run(self) + + def make_release_tree(self, base_dir, files): + root = get_root() + cfg = get_config_from_root(root) + _sdist.make_release_tree(self, base_dir, files) + # now locate _version.py in the new base_dir directory + # (remembering that it may be a hardlink) and replace it with an + # updated value + target_versionfile = os.path.join(base_dir, cfg.versionfile_source) print("UPDATING %s" % target_versionfile) - os.unlink(target_versionfile) - f = open(target_versionfile, "w") - f.write(SHORT_VERSION_PY % versions) - f.close() - _build_exe.run(self) - os.unlink(target_versionfile) - f = open(versionfile_source, "w") - f.write(LONG_VERSION_PY % {"DOLLAR": "$", - "TAG_PREFIX": tag_prefix, - "PARENTDIR_PREFIX": parentdir_prefix, - "VERSIONFILE_SOURCE": versionfile_source, - "LOOKUPFILE": '"%s"' % lookupfile if lookupfile is not None else "None", - }) - f.close() - -class cmd_sdist(_sdist): - def run(self): - versions = get_versions(verbose=True) - self._versioneer_generated_versions = versions - # unless we update this, the command will keep using the old version - self.distribution.metadata.version = versions["version"] - return _sdist.run(self) - - def make_release_tree(self, base_dir, files): - _sdist.make_release_tree(self, base_dir, files) - # now locate _version.py in the new base_dir directory (remembering - # that it may be a hardlink) and replace it with an updated value - target_versionfile = os.path.join(base_dir, versionfile_source) - print("UPDATING %s" % target_versionfile) - os.unlink(target_versionfile) - f = open(target_versionfile, "w") - f.write(SHORT_VERSION_PY % self._versioneer_generated_versions) - f.close() + write_to_version_file(target_versionfile, + self._versioneer_generated_versions) + cmds["sdist"] = cmd_sdist + + return cmds + + +CONFIG_ERROR = """ +setup.cfg is missing the necessary Versioneer configuration. You need +a section like: + + [versioneer] + VCS = git + style = pep440 + versionfile_source = src/myproject/_version.py + versionfile_build = myproject/_version.py + tag_prefix = + parentdir_prefix = myproject- + +You will also need to edit your setup.py to use the results: + + import versioneer + setup(version=versioneer.get_version(), + cmdclass=versioneer.get_cmdclass(), ...) + +Please read the docstring in ./versioneer.py for configuration instructions, +edit setup.cfg, and re-run the installer or 'python versioneer.py setup'. +""" + +SAMPLE_CONFIG = """ +# See the docstring in versioneer.py for instructions. Note that you must +# re-run 'versioneer.py setup' after changing this section, and commit the +# resulting files. + +[versioneer] +#VCS = git +#style = pep440 +#versionfile_source = +#versionfile_build = +#tag_prefix = +#parentdir_prefix = +#lookupfile = + +""" INIT_PY_SNIPPET = """ from ._version import get_versions @@ -1039,85 +1855,127 @@ def make_release_tree(self, base_dir, files): del get_versions """ -class cmd_update_files(Command): - description = "install/upgrade Versioneer files: __init__.py SRC/_version.py" - user_options = [] - boolean_options = [] - def initialize_options(self): - pass - def finalize_options(self): - pass - def run(self): - print(" creating %s" % versionfile_source) - f = open(versionfile_source, "w") - f.write(LONG_VERSION_PY % {"DOLLAR": "$", - "TAG_PREFIX": tag_prefix, - "PARENTDIR_PREFIX": parentdir_prefix, - "VERSIONFILE_SOURCE": versionfile_source, - "LOOKUPFILE": '"%s"' % lookupfile if lookupfile is not None else "None", - }) - f.close() - ipy = os.path.join(os.path.dirname(versionfile_source), "__init__.py") +def do_setup(): + root = get_root() + try: + cfg = get_config_from_root(root) + except (EnvironmentError, configparser.NoSectionError, + configparser.NoOptionError) as e: + if isinstance(e, (EnvironmentError, configparser.NoSectionError)): + print("Adding sample versioneer config to setup.cfg", + file=sys.stderr) + with open(os.path.join(root, "setup.cfg"), "a") as f: + f.write(SAMPLE_CONFIG) + print(CONFIG_ERROR, file=sys.stderr) + return 1 + + print(" creating %s" % cfg.versionfile_source) + with open(cfg.versionfile_source, "w") as f: + LONG = LONG_VERSION_PY[cfg.VCS] + f.write(LONG % {"DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + }) + + ipy = os.path.join(os.path.dirname(cfg.versionfile_source), + "__init__.py") + if os.path.exists(ipy): try: - old = open(ipy, "r").read() + with open(ipy, "r") as f: + old = f.read() except EnvironmentError: old = "" if INIT_PY_SNIPPET not in old: print(" appending to %s" % ipy) - f = open(ipy, "a") - f.write(INIT_PY_SNIPPET) - f.close() + with open(ipy, "a") as f: + f.write(INIT_PY_SNIPPET) else: print(" %s unmodified" % ipy) - - # Make sure both the top-level "versioneer.py" and versionfile_source - # (PKG/_version.py, used by runtime code) are in MANIFEST.in, so - # they'll be copied into source distributions. Pip won't be able to - # install the package without this. - manifest_in = os.path.join(get_root(), "MANIFEST.in") - simple_includes = set() - try: - for line in open(manifest_in, "r").readlines(): + else: + print(" %s doesn't exist, ok" % ipy) + ipy = None + + # Make sure both the top-level "versioneer.py" and versionfile_source + # (PKG/_version.py, used by runtime code) are in MANIFEST.in, so + # they'll be copied into source distributions. Pip won't be able to + # install the package without this. + manifest_in = os.path.join(root, "MANIFEST.in") + simple_includes = set() + try: + with open(manifest_in, "r") as f: + for line in f: if line.startswith("include "): for include in line.split()[1:]: simple_includes.add(include) - except EnvironmentError: - pass - # That doesn't cover everything MANIFEST.in can do - # (http://docs.python.org/2/distutils/sourcedist.html#commands), so - # it might give some false negatives. Appending redundant 'include' - # lines is safe, though. - if "versioneer.py" not in simple_includes: - print(" appending 'versioneer.py' to MANIFEST.in") - f = open(manifest_in, "a") + except EnvironmentError: + pass + # That doesn't cover everything MANIFEST.in can do + # (http://docs.python.org/2/distutils/sourcedist.html#commands), so + # it might give some false negatives. Appending redundant 'include' + # lines is safe, though. + if "versioneer.py" not in simple_includes: + print(" appending 'versioneer.py' to MANIFEST.in") + with open(manifest_in, "a") as f: f.write("include versioneer.py\n") - f.close() - else: - print(" 'versioneer.py' already in MANIFEST.in") - if versionfile_source not in simple_includes: - print(" appending versionfile_source ('%s') to MANIFEST.in" % - versionfile_source) - f = open(manifest_in, "a") - f.write("include %s\n" % versionfile_source) - f.close() - else: - print(" versionfile_source already in MANIFEST.in") + else: + print(" 'versioneer.py' already in MANIFEST.in") + if cfg.versionfile_source not in simple_includes: + print(" appending versionfile_source ('%s') to MANIFEST.in" % + cfg.versionfile_source) + with open(manifest_in, "a") as f: + f.write("include %s\n" % cfg.versionfile_source) + else: + print(" versionfile_source already in MANIFEST.in") - # Make VCS-specific changes. For git, this means creating/changing - # .gitattributes to mark _version.py for export-time keyword - # substitution. - do_vcs_install(manifest_in, versionfile_source, ipy) + # Make VCS-specific changes. For git, this means creating/changing + # .gitattributes to mark _version.py for export-time keyword + # substitution. + do_vcs_install(manifest_in, cfg.versionfile_source, ipy) + return 0 -def get_cmdclass(): - cmds = {'version': cmd_version, - 'versioneer': cmd_update_files, - 'build': cmd_build, - 'build_py': cmd_build_py, - 'sdist': cmd_sdist, - } - if 'cx_Freeze' in sys.modules: # cx_freeze enabled? - cmds['build_exe'] = cmd_build_exe - del cmds['build'] - return cmds +def scan_setup_py(): + found = set() + setters = False + errors = 0 + with open("setup.py", "r") as f: + for line in f.readlines(): + if "import versioneer" in line: + found.add("import") + if "versioneer.get_cmdclass()" in line: + found.add("cmdclass") + if "versioneer.get_version()" in line: + found.add("get_version") + if "versioneer.VCS" in line: + setters = True + if "versioneer.versionfile_source" in line: + setters = True + if len(found) != 3: + print("") + print("Your setup.py appears to be missing some important items") + print("(but I might be wrong). Please make sure it has something") + print("roughly like the following:") + print("") + print(" import versioneer") + print(" setup( version=versioneer.get_version(),") + print(" cmdclass=versioneer.get_cmdclass(), ...)") + print("") + errors += 1 + if setters: + print("You should remove lines like 'versioneer.VCS = ' and") + print("'versioneer.versionfile_source = ' . This configuration") + print("now lives in setup.cfg, and should be removed from setup.py") + print("") + errors += 1 + return errors + +if __name__ == "__main__": + cmd = sys.argv[1] + if cmd == "setup": + errors = do_setup() + errors += scan_setup_py() + if errors: + sys.exit(1) From f05f850175a0f2f2eae0dfc8940ef9bba18fad9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Tue, 30 Jun 2015 14:57:39 +0200 Subject: [PATCH 04/66] Switched to new version of versioneer Versions should now be PEP440 compatible. (cherry picked from commit 711810d) --- .versioneer-lookup | 6 ++--- src/octoprint/_version.py | 24 +++++++++++++++++++ versioneer.py | 50 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 77 insertions(+), 3 deletions(-) diff --git a/.versioneer-lookup b/.versioneer-lookup index 78d848c94..0ffcdd877 100644 --- a/.versioneer-lookup +++ b/.versioneer-lookup @@ -12,11 +12,11 @@ master # maintenance is currently the branch for preparation of maintenance release 1.2.8 # so are any fix/... branches -maintenance 1.2.8-dev 6c622f7c4332b71c6ece59552ffc87c146155c84 -fix/.* 1.2.8-dev 6c622f7c4332b71c6ece59552ffc87c146155c84 +maintenance 1.2.8 6c622f7c4332b71c6ece59552ffc87c146155c84 pep440-dev +fix/.* 1.2.8 6c622f7c4332b71c6ece59552ffc87c146155c84 pep440-dev # Special case disconnected checkouts, e.g. 'git checkout ' \(detached.* # every other branch is a development branch and thus gets resolved to 1.3.0-dev for now -.* 1.3.0 198d3450d94be1a2 pep440-pre +.* 1.3.0 198d3450d94be1a2 pep440-dev diff --git a/src/octoprint/_version.py b/src/octoprint/_version.py index b96249a5b..8064b06f8 100644 --- a/src/octoprint/_version.py +++ b/src/octoprint/_version.py @@ -447,6 +447,28 @@ def render_pep440_post(pieces): return rendered +def render_pep440_dev(pieces): + """ TAG.dev[DISTANCE]+gHEX[.dirty] + + Exceptions: + 1: no tags. 0.devDISTANCE+gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += ".dev%d" % pieces["distance"] + else: + rendered += ".dev" + rendered += plus_or_dot(pieces) + else: + # exception #1 + rendered = "0.dev%d" % pieces["distance"] + rendered += "+" + rendered += "g%s" % pieces["short"] + if pieces["dirty"]: + rendered += ".dirty" + return rendered + def render_pep440_old(pieces): """TAG[.postDISTANCE[.dev0]] . @@ -531,6 +553,8 @@ def render(pieces, style): rendered = render_pep440_post(pieces) elif style == "pep440-old": rendered = render_pep440_old(pieces) + elif style == "pep440-dev": + rendered = render_pep440_dev(pieces) elif style == "git-describe": rendered = render_git_describe(pieces) elif style == "git-describe-long": diff --git a/versioneer.py b/versioneer.py index 51c6327de..f9cf9443c 100644 --- a/versioneer.py +++ b/versioneer.py @@ -919,6 +919,28 @@ def render_pep440_post(pieces): return rendered +def render_pep440_dev(pieces): + """ TAG.dev[DISTANCE]+gHEX[.dirty] + + Exceptions: + 1: no tags. 0.devDISTANCE+gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += ".dev%%d" %% pieces["distance"] + else: + rendered += ".dev" + rendered += plus_or_dot(pieces) + else: + # exception #1 + rendered = "0.dev%%d" %% pieces["distance"] + rendered += "+" + rendered += "g%%s" %% pieces["short"] + if pieces["dirty"]: + rendered += ".dirty" + return rendered + def render_pep440_old(pieces): """TAG[.postDISTANCE[.dev0]] . @@ -1003,6 +1025,8 @@ def render(pieces, style): rendered = render_pep440_post(pieces) elif style == "pep440-old": rendered = render_pep440_old(pieces) + elif style == "pep440-dev": + rendered = render_pep440_dev(pieces) elif style == "git-describe": rendered = render_git_describe(pieces) elif style == "git-describe-long": @@ -1490,6 +1514,28 @@ def render_pep440_post(pieces): return rendered +def render_pep440_dev(pieces): + """ TAG.dev[DISTANCE]+gHEX[.dirty] + + Exceptions: + 1: no tags. 0.devDISTANCE+gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += ".dev%d" % pieces["distance"] + else: + rendered += ".dev" + rendered += plus_or_dot(pieces) + else: + # exception #1 + rendered = "0.dev%d" % pieces["distance"] + rendered += "+" + rendered += "g%s" % pieces["short"] + if pieces["dirty"]: + rendered += ".dirty" + return rendered + def render_pep440_old(pieces): """TAG[.postDISTANCE[.dev0]] . @@ -1574,6 +1620,8 @@ def render(pieces, style): rendered = render_pep440_post(pieces) elif style == "pep440-old": rendered = render_pep440_old(pieces) + elif style == "pep440-dev": + rendered = render_pep440_dev(pieces) elif style == "git-describe": rendered = render_git_describe(pieces) elif style == "git-describe-long": @@ -1722,6 +1770,8 @@ def run(self): vers = get_versions(verbose=True) print("Version: %s" % vers["version"]) print(" full-revisionid: %s" % vers.get("full-revisionid")) + if "branch" in vers: + print(" branch: %s" % vers["branch"]) print(" dirty: %s" % vers.get("dirty")) if vers["error"]: print(" error: %s" % vers["error"]) From e875d042a7f77b2dd860b4dc4df3e565fc052e08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Tue, 30 Jun 2015 15:15:49 +0200 Subject: [PATCH 05/66] Refreshed versioneer again (cherry picked from commit 7cd0adb) --- src/octoprint/_version.py | 95 ++++++++++++------ versioneer.py | 201 +++++++++++++++++++++++++++----------- 2 files changed, 208 insertions(+), 88 deletions(-) diff --git a/src/octoprint/_version.py b/src/octoprint/_version.py index 8064b06f8..60161f3c8 100644 --- a/src/octoprint/_version.py +++ b/src/octoprint/_version.py @@ -172,7 +172,8 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): if verbose: print("discarding '%s', no digits" % ",".join(refs-tags)) - branches = [r for r in refs if not r.startswith(TAG) and r != "HEAD" and not r.startswith("refs/")] + branches = [r for r in refs if not r.startswith(TAG) + and r != "HEAD" and not r.startswith("refs/")] if verbose: print("likely branches: %s" % ",".join(sorted(branches))) branch = None @@ -189,9 +190,8 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): print("picking %s" % r) result = {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None - } + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None} if branch is not None: result["branch"] = branch return result @@ -250,7 +250,9 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): git_describe = git_describe[:git_describe.rindex("-dirty")] # figure out our branch - abbrev_ref_out = run_command(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], cwd=root) + abbrev_ref_out = run_command(GITS, + ["rev-parse", "--abbrev-ref", "HEAD"], + cwd=root) if abbrev_ref_out is not None: pieces["branch"] = abbrev_ref_out.strip() @@ -294,6 +296,11 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): @register_vcs_handler("git", "parse_lookup_file") def git_parse_lookup_file(path): + """Parse a versioneer lookup file. + + This file allows definition of branch specific data like virtual tags or + custom styles to use for version rendering. + """ if not os.path.exists(path): return [] @@ -304,22 +311,39 @@ def git_parse_lookup_file(path): if '#' in line: line = line[:line.rindex("#")] line = line.strip() + if not line: + continue + try: - split_line = line.split() - if len(split_line) == 3: - pattern, tag, ref_commit = split_line - lookup.append([re.compile(pattern), tag, ref_commit, None]) + split_line = map(lambda x: x.strip(), line.split()) + if not len(split_line): + continue + + matcher = re.compile(split_line[0]) + + if len(split_line) == 1: + entry = [matcher, None, None, None] + elif len(split_line) == 2: + render = split_line[1] + entry = [matcher, render, None, None] + elif len(split_line) == 3: + tag, ref_commit = split_line[1:] + entry = [matcher, None, tag, ref_commit] elif len(split_line) == 4: - pattern, tag, ref_commit, render = split_line - lookup.append([re.compile(pattern), tag, ref_commit, render]) - elif len(split_line) >= 1: - lookup.append([re.compile(split_line[0]), None, None, None]) + tag, ref_commit, render = split_line[1:] + entry = [matcher, render, tag, ref_commit] + else: + continue + + lookup.append(entry) except: break return lookup + @register_vcs_handler("git", "pieces_from_lookup") def git_pieces_from_lookup(lookup, root, verbose, run_command=run_command): + """Extract version information based on provided lookup data.""" GITS = ["git"] if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] @@ -330,27 +354,40 @@ def git_pieces_from_lookup(lookup, root, verbose, run_command=run_command): raise NotThisMethod("git rev-parse --abbrev-ref HEAD failed") current_branch = stdout.strip() - for matcher, tag, ref_commit, render in lookup: + for matcher, render, tag, ref_commit in lookup: if matcher.match(current_branch): if tag is None or ref_commit is None: - raise NotThisMethod("tag or ref_commit is unset for this branch") + raise NotThisMethod("tag or ref_commit is unset for " + "this branch") - stdout = run_command(GITS, ["rev-list", "%s..HEAD" % ref_commit, "--count"], cwd=root) + stdout = run_command(GITS, + ["rev-list", "%s..HEAD" % ref_commit, + "--count"], + cwd=root) if stdout is None: - raise NotThisMethod("git rev-list %s..HEAD --count failed" % ref_commit) + raise NotThisMethod("git rev-list %s..HEAD " + "--count failed" % ref_commit) try: num_commits = int(stdout.strip()) except ValueError: - raise NotThisMethod("git rev-list %s..HEAD --count didn't return a valid number" % ref_commit) + raise NotThisMethod("git rev-list %s..HEAD --count didn't " + "return a valid number" % ref_commit) - stdout =run_command(GITS, ["rev-parse", "--short", "HEAD"], cwd=root) + stdout = run_command(GITS, + ["rev-parse", "--short", "HEAD"], + cwd=root) if stdout is None: - raise NotThisMethod("git describe rev-parse --short HEAD failed") + raise NotThisMethod("git describe rev-parse " + "--short HEAD failed") short_hash = stdout.strip() - stdout = run_command(GITS, ["describe", "--tags", "--dirty", "--always"], cwd=root) + stdout = run_command(GITS, + ["describe", "--tags", + "--dirty", "--always"], + cwd=root) if stdout is None: - raise NotThisMethod("git describe --tags --dirty --always failed") + raise NotThisMethod("git describe --tags --dirty " + "--always failed") dirty = stdout.strip().endswith("-dirty") stdout = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) @@ -448,7 +485,7 @@ def render_pep440_post(pieces): def render_pep440_dev(pieces): - """ TAG.dev[DISTANCE]+gHEX[.dirty] + """TAG[.devDISTANCE]+gHEX[.dirty] . Exceptions: 1: no tags. 0.devDISTANCE+gHEX[.dirty] @@ -457,8 +494,6 @@ def render_pep440_dev(pieces): rendered = pieces["closest-tag"] if pieces["distance"]: rendered += ".dev%d" % pieces["distance"] - else: - rendered += ".dev" rendered += plus_or_dot(pieces) else: # exception #1 @@ -469,6 +504,7 @@ def render_pep440_dev(pieces): rendered += ".dirty" return rendered + def render_pep440_old(pieces): """TAG[.postDISTANCE[.dev0]] . @@ -563,7 +599,7 @@ def render(pieces, style): raise ValueError("unknown style '%s'" % style) result = {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None} + "dirty": pieces["dirty"], "error": None} if "branch" in pieces and pieces["branch"] is not None: result["branch"] = pieces["branch"] return result @@ -597,10 +633,11 @@ def get_versions(): "dirty": None, "error": "unable to find root of source tree"} - lookupfile = os.path.join(root, cfg.lookupfile if cfg.lookupfile is not None else ".versioneer-lookup") - if os.path.exists(lookupfile): + lookupfile = cfg.lookupfile if cfg.lookupfile is not None else ".versioneer-lookup" + lookuppath = os.path.join(root, lookupfile) + if os.path.exists(lookuppath): try: - lookup_data = git_parse_lookup_file(lookupfile) + lookup_data = git_parse_lookup_file(lookuppath) pieces = git_pieces_from_lookup(lookup_data, root, verbose) return render(pieces, cfg.style) except NotThisMethod: diff --git a/versioneer.py b/versioneer.py index f9cf9443c..bfbefda1c 100644 --- a/versioneer.py +++ b/versioneer.py @@ -235,12 +235,19 @@ useful to throw an exception in setup.py if this is set, to avoid e.g. creating tarballs with a version string of "unknown". +If the underlying VCS supports it and that information is available, this will +also be included: + +* `['branch']`: A string with the VCS branch name the version was built on. + Some variants are more useful than others. Including `full-revisionid` in a bug report should allow developers to reconstruct the exact code being tested (or indicate the presence of local changes that should be shared with the developers). `version` is suitable for display in an "about" box or a CLI `--version` output: it can be easily compared against release notes and lists -of bugs fixed in various releases. +of bugs fixed in various releases. Augmenting that with the `branch` +information if it is available will give additional hints during bug reporting +what kind of setup a user was running. The installer adds the following text to your `__init__.py` to place a basic version in `YOURPROJECT.__version__`: @@ -644,7 +651,8 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): if verbose: print("discarding '%%s', no digits" %% ",".join(refs-tags)) - branches = [r for r in refs if not r.startswith(TAG) and r != "HEAD" and not r.startswith("refs/")] + branches = [r for r in refs if not r.startswith(TAG) + and r != "HEAD" and not r.startswith("refs/")] if verbose: print("likely branches: %%s" %% ",".join(sorted(branches))) branch = None @@ -661,9 +669,8 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): print("picking %%s" %% r) result = {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None - } + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None} if branch is not None: result["branch"] = branch return result @@ -722,7 +729,9 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): git_describe = git_describe[:git_describe.rindex("-dirty")] # figure out our branch - abbrev_ref_out = run_command(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], cwd=root) + abbrev_ref_out = run_command(GITS, + ["rev-parse", "--abbrev-ref", "HEAD"], + cwd=root) if abbrev_ref_out is not None: pieces["branch"] = abbrev_ref_out.strip() @@ -766,6 +775,11 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): @register_vcs_handler("git", "parse_lookup_file") def git_parse_lookup_file(path): + """Parse a versioneer lookup file. + + This file allows definition of branch specific data like virtual tags or + custom styles to use for version rendering. + """ if not os.path.exists(path): return [] @@ -776,22 +790,39 @@ def git_parse_lookup_file(path): if '#' in line: line = line[:line.rindex("#")] line = line.strip() + if not line: + continue + try: - split_line = line.split() - if len(split_line) == 3: - pattern, tag, ref_commit = split_line - lookup.append([re.compile(pattern), tag, ref_commit, None]) + split_line = map(lambda x: x.strip(), line.split()) + if not len(split_line): + continue + + matcher = re.compile(split_line[0]) + + if len(split_line) == 1: + entry = [matcher, None, None, None] + elif len(split_line) == 2: + render = split_line[1] + entry = [matcher, render, None, None] + elif len(split_line) == 3: + tag, ref_commit = split_line[1:] + entry = [matcher, None, tag, ref_commit] elif len(split_line) == 4: - pattern, tag, ref_commit, render = split_line - lookup.append([re.compile(pattern), tag, ref_commit, render]) - elif len(split_line) >= 1: - lookup.append([re.compile(split_line[0]), None, None, None]) + tag, ref_commit, render = split_line[1:] + entry = [matcher, render, tag, ref_commit] + else: + continue + + lookup.append(entry) except: break return lookup + @register_vcs_handler("git", "pieces_from_lookup") def git_pieces_from_lookup(lookup, root, verbose, run_command=run_command): + """Extract version information based on provided lookup data.""" GITS = ["git"] if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] @@ -802,27 +833,40 @@ def git_pieces_from_lookup(lookup, root, verbose, run_command=run_command): raise NotThisMethod("git rev-parse --abbrev-ref HEAD failed") current_branch = stdout.strip() - for matcher, tag, ref_commit, render in lookup: + for matcher, render, tag, ref_commit in lookup: if matcher.match(current_branch): if tag is None or ref_commit is None: - raise NotThisMethod("tag or ref_commit is unset for this branch") + raise NotThisMethod("tag or ref_commit is unset for " + "this branch") - stdout = run_command(GITS, ["rev-list", "%%s..HEAD" %% ref_commit, "--count"], cwd=root) + stdout = run_command(GITS, + ["rev-list", "%%s..HEAD" %% ref_commit, + "--count"], + cwd=root) if stdout is None: - raise NotThisMethod("git rev-list %%s..HEAD --count failed" %% ref_commit) + raise NotThisMethod("git rev-list %%s..HEAD " + "--count failed" %% ref_commit) try: num_commits = int(stdout.strip()) except ValueError: - raise NotThisMethod("git rev-list %%s..HEAD --count didn't return a valid number" %% ref_commit) + raise NotThisMethod("git rev-list %%s..HEAD --count didn't " + "return a valid number" %% ref_commit) - stdout =run_command(GITS, ["rev-parse", "--short", "HEAD"], cwd=root) + stdout = run_command(GITS, + ["rev-parse", "--short", "HEAD"], + cwd=root) if stdout is None: - raise NotThisMethod("git describe rev-parse --short HEAD failed") + raise NotThisMethod("git describe rev-parse " + "--short HEAD failed") short_hash = stdout.strip() - stdout = run_command(GITS, ["describe", "--tags", "--dirty", "--always"], cwd=root) + stdout = run_command(GITS, + ["describe", "--tags", + "--dirty", "--always"], + cwd=root) if stdout is None: - raise NotThisMethod("git describe --tags --dirty --always failed") + raise NotThisMethod("git describe --tags --dirty " + "--always failed") dirty = stdout.strip().endswith("-dirty") stdout = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) @@ -920,7 +964,7 @@ def render_pep440_post(pieces): def render_pep440_dev(pieces): - """ TAG.dev[DISTANCE]+gHEX[.dirty] + """TAG[.devDISTANCE]+gHEX[.dirty] . Exceptions: 1: no tags. 0.devDISTANCE+gHEX[.dirty] @@ -929,8 +973,6 @@ def render_pep440_dev(pieces): rendered = pieces["closest-tag"] if pieces["distance"]: rendered += ".dev%%d" %% pieces["distance"] - else: - rendered += ".dev" rendered += plus_or_dot(pieces) else: # exception #1 @@ -941,6 +983,7 @@ def render_pep440_dev(pieces): rendered += ".dirty" return rendered + def render_pep440_old(pieces): """TAG[.postDISTANCE[.dev0]] . @@ -1035,7 +1078,7 @@ def render(pieces, style): raise ValueError("unknown style '%%s'" %% style) result = {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None} + "dirty": pieces["dirty"], "error": None} if "branch" in pieces and pieces["branch"] is not None: result["branch"] = pieces["branch"] return result @@ -1069,10 +1112,12 @@ def get_versions(): "dirty": None, "error": "unable to find root of source tree"} - lookupfile = os.path.join(root, cfg.lookupfile if cfg.lookupfile is not None else ".versioneer-lookup") - if os.path.exists(lookupfile): + lookupfile = cfg.lookupfile if cfg.lookupfile is not None \ + else ".versioneer-lookup" + lookuppath = os.path.join(root, lookupfile) + if os.path.exists(lookuppath): try: - lookup_data = git_parse_lookup_file(lookupfile) + lookup_data = git_parse_lookup_file(lookuppath) pieces = git_pieces_from_lookup(lookup_data, root, verbose) return render(pieces, cfg.style) except NotThisMethod: @@ -1148,7 +1193,8 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): if verbose: print("discarding '%s', no digits" % ",".join(refs-tags)) - branches = [r for r in refs if not r.startswith(TAG) and r != "HEAD" and not r.startswith("refs/")] + branches = [r for r in refs if not r.startswith(TAG) + and r != "HEAD" and not r.startswith("refs/")] if verbose: print("likely branches: %s" % ",".join(sorted(branches))) branch = None @@ -1165,9 +1211,8 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): print("picking %s" % r) result = {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None - } + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None} if branch is not None: result["branch"] = branch return result @@ -1226,7 +1271,9 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): git_describe = git_describe[:git_describe.rindex("-dirty")] # figure out our branch - abbrev_ref_out = run_command(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], cwd=root) + abbrev_ref_out = run_command(GITS, + ["rev-parse", "--abbrev-ref", "HEAD"], + cwd=root) if abbrev_ref_out is not None: pieces["branch"] = abbrev_ref_out.strip() @@ -1270,6 +1317,11 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): @register_vcs_handler("git", "parse_lookup_file") def git_parse_lookup_file(path): + """Parse a versioneer lookup file. + + This file allows definition of branch specific data like virtual tags or + custom styles to use for version rendering. + """ if not os.path.exists(path): return [] @@ -1280,22 +1332,39 @@ def git_parse_lookup_file(path): if '#' in line: line = line[:line.rindex("#")] line = line.strip() + if not line: + continue + try: - split_line = line.split() - if len(split_line) == 3: - pattern, tag, ref_commit = split_line - lookup.append([re.compile(pattern), tag, ref_commit, None]) + split_line = map(lambda x: x.strip(), line.split()) + if not len(split_line): + continue + + matcher = re.compile(split_line[0]) + + if len(split_line) == 1: + entry = [matcher, None, None, None] + elif len(split_line) == 2: + render = split_line[1] + entry = [matcher, render, None, None] + elif len(split_line) == 3: + tag, ref_commit = split_line[1:] + entry = [matcher, None, tag, ref_commit] elif len(split_line) == 4: - pattern, tag, ref_commit, render = split_line - lookup.append([re.compile(pattern), tag, ref_commit, render]) - elif len(split_line) >= 1: - lookup.append([re.compile(split_line[0]), None, None, None]) + tag, ref_commit, render = split_line[1:] + entry = [matcher, render, tag, ref_commit] + else: + continue + + lookup.append(entry) except: break return lookup + @register_vcs_handler("git", "pieces_from_lookup") def git_pieces_from_lookup(lookup, root, verbose, run_command=run_command): + """Extract version information based on provided lookup data.""" GITS = ["git"] if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] @@ -1306,27 +1375,40 @@ def git_pieces_from_lookup(lookup, root, verbose, run_command=run_command): raise NotThisMethod("git rev-parse --abbrev-ref HEAD failed") current_branch = stdout.strip() - for matcher, tag, ref_commit, render in lookup: + for matcher, render, tag, ref_commit in lookup: if matcher.match(current_branch): if tag is None or ref_commit is None: - raise NotThisMethod("tag or ref_commit is unset for this branch") + raise NotThisMethod("tag or ref_commit is unset for " + "this branch") - stdout = run_command(GITS, ["rev-list", "%s..HEAD" % ref_commit, "--count"], cwd=root) + stdout = run_command(GITS, + ["rev-list", "%s..HEAD" % ref_commit, + "--count"], + cwd=root) if stdout is None: - raise NotThisMethod("git rev-list %s..HEAD --count failed" % ref_commit) + raise NotThisMethod("git rev-list %s..HEAD " + "--count failed" % ref_commit) try: num_commits = int(stdout.strip()) except ValueError: - raise NotThisMethod("git rev-list %s..HEAD --count didn't return a valid number" % ref_commit) + raise NotThisMethod("git rev-list %s..HEAD --count didn't " + "return a valid number" % ref_commit) - stdout =run_command(GITS, ["rev-parse", "--short", "HEAD"], cwd=root) + stdout = run_command(GITS, + ["rev-parse", "--short", "HEAD"], + cwd=root) if stdout is None: - raise NotThisMethod("git describe rev-parse --short HEAD failed") + raise NotThisMethod("git describe rev-parse " + "--short HEAD failed") short_hash = stdout.strip() - stdout = run_command(GITS, ["describe", "--tags", "--dirty", "--always"], cwd=root) + stdout = run_command(GITS, + ["describe", "--tags", + "--dirty", "--always"], + cwd=root) if stdout is None: - raise NotThisMethod("git describe --tags --dirty --always failed") + raise NotThisMethod("git describe --tags --dirty " + "--always failed") dirty = stdout.strip().endswith("-dirty") stdout = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) @@ -1515,7 +1597,7 @@ def render_pep440_post(pieces): def render_pep440_dev(pieces): - """ TAG.dev[DISTANCE]+gHEX[.dirty] + """TAG[.devDISTANCE]+gHEX[.dirty] . Exceptions: 1: no tags. 0.devDISTANCE+gHEX[.dirty] @@ -1524,8 +1606,6 @@ def render_pep440_dev(pieces): rendered = pieces["closest-tag"] if pieces["distance"]: rendered += ".dev%d" % pieces["distance"] - else: - rendered += ".dev" rendered += plus_or_dot(pieces) else: # exception #1 @@ -1536,6 +1616,7 @@ def render_pep440_dev(pieces): rendered += ".dirty" return rendered + def render_pep440_old(pieces): """TAG[.postDISTANCE[.dev0]] . @@ -1630,7 +1711,7 @@ def render(pieces, style): raise ValueError("unknown style '%s'" % style) result = {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None} + "dirty": pieces["dirty"], "error": None} if "branch" in pieces and pieces["branch"] is not None: result["branch"] = pieces["branch"] return result @@ -1686,13 +1767,15 @@ def get_versions(verbose=False): except NotThisMethod: pass - lookupfile = os.path.join(root, cfg.lookupfile if cfg.lookupfile is not None else ".versioneer-lookup") - if os.path.exists(lookupfile): + lookupfile = cfg.lookupfile if cfg.lookupfile is not None \ + else ".versioneer-lookup" + lookuppath = os.path.join(root, lookupfile) + if os.path.exists(lookuppath): parse_lookup_file_f = handlers.get("parse_lookup_file") versions_from_lookup_f = handlers.get("pieces_from_lookup") if parse_lookup_file_f and versions_from_lookup_f: try: - lookup_data = parse_lookup_file_f(lookupfile) + lookup_data = parse_lookup_file_f(lookuppath) pieces = versions_from_lookup_f(lookup_data, root, verbose) ver = render(pieces, cfg.style) if verbose: From 2e566c5ee14d062e3b578d577662dc21af7d6643 Mon Sep 17 00:00:00 2001 From: Koen Kooi Date: Tue, 1 Sep 2015 08:58:57 +0200 Subject: [PATCH 06/66] versioneer lookup: add support for detached checkouts When doing 'git checkout ' or 'git checkout ' there will be no branch info: $ git branch * (detached from 96fc70b) After whitelisting '\(detached.*' we get: $ python setup.py version running version got version from git {'version': '1.2.5-1-gfd73e02', 'branch': 'HEAD', 'full': 'fd73e02f4fa9f286407ad688feeeee3ebaffea60'} Version is currently: 1.2.5-1-gfd73e02 The '-1-gfd73e02' is this commit on top of the 1.2.5 tag. While detached checkouts should be discouraged some buildsystems (e.g. OpenEmbedded) have no real way to avoid it, which is what this patch fixes. Actual people doing a checkout should use a branch. Signed-off-by: Koen Kooi (cherry picked from commit 987e637) --- .versioneer-lookup | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.versioneer-lookup b/.versioneer-lookup index 0ffcdd877..203936b49 100644 --- a/.versioneer-lookup +++ b/.versioneer-lookup @@ -18,5 +18,8 @@ fix/.* 1.2.8 6c622f7c4332b71c6ece59552ffc87c146155c84 pep440-dev # Special case disconnected checkouts, e.g. 'git checkout ' \(detached.* +# Special case disconnected checkouts, e.g. 'git checkout ' +(detached.* + # every other branch is a development branch and thus gets resolved to 1.3.0-dev for now .* 1.3.0 198d3450d94be1a2 pep440-dev From 729d23b2c0a325da36b354433b0358732469a134 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Wed, 2 Sep 2015 10:03:39 +0200 Subject: [PATCH 07/66] Fixed versioneer lookup file again Runaway regex ;) (cherry picked from commit 78277ac) --- .versioneer-lookup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.versioneer-lookup b/.versioneer-lookup index 203936b49..11b214aca 100644 --- a/.versioneer-lookup +++ b/.versioneer-lookup @@ -19,7 +19,7 @@ fix/.* 1.2.8 6c622f7c4332b71c6ece59552ffc87c146155c84 pep440-dev \(detached.* # Special case disconnected checkouts, e.g. 'git checkout ' -(detached.* +\(detached.* # every other branch is a development branch and thus gets resolved to 1.3.0-dev for now .* 1.3.0 198d3450d94be1a2 pep440-dev From 528192b3c0466cead641b5e8d41a5a3d90a4528a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Fri, 23 Oct 2015 09:55:14 +0200 Subject: [PATCH 08/66] Fixed a merge error while migrating versioneer update --- .versioneer-lookup | 3 --- 1 file changed, 3 deletions(-) diff --git a/.versioneer-lookup b/.versioneer-lookup index 11b214aca..0ffcdd877 100644 --- a/.versioneer-lookup +++ b/.versioneer-lookup @@ -18,8 +18,5 @@ fix/.* 1.2.8 6c622f7c4332b71c6ece59552ffc87c146155c84 pep440-dev # Special case disconnected checkouts, e.g. 'git checkout ' \(detached.* -# Special case disconnected checkouts, e.g. 'git checkout ' -\(detached.* - # every other branch is a development branch and thus gets resolved to 1.3.0-dev for now .* 1.3.0 198d3450d94be1a2 pep440-dev From 05d6cac67a30f204fdb9d944759dc90fd8110eeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Fri, 23 Oct 2015 10:02:05 +0200 Subject: [PATCH 09/66] Define where our README is located so that it gets rendered correctly on pypi --- setup.cfg | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/setup.cfg b/setup.cfg index e814cb446..e197177a3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,7 +1,5 @@ - -# See the docstring in versioneer.py for instructions. Note that you must -# re-run 'versioneer.py setup' after changing this section, and commit the -# resulting files. +[metadata] +description-file = README.md [versioneer] VCS = git From bdadab699364f0132249024e53a566a1cfd7639e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Fri, 23 Oct 2015 10:07:15 +0200 Subject: [PATCH 10/66] We want pep440-post by default --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index e197177a3..cbda2efbf 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,7 +3,7 @@ description-file = README.md [versioneer] VCS = git -style = pep440 +style = pep440-post versionfile_source = src/octoprint/_version.py versionfile_build = octoprint/_version.py tag_prefix = From ce00a069fa4472f2803a1e590efed6b480368e44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Fri, 23 Oct 2015 12:08:36 +0200 Subject: [PATCH 11/66] Added conversion for README.md to rst for PyPi --- setup.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 878c18b4e..e8eb944da 100644 --- a/setup.py +++ b/setup.py @@ -48,6 +48,9 @@ "sphinx>=1.3", "sphinxcontrib-httpdomain", "sphinx_rtd_theme" + + # PyPi upload related + "pypandoc" ], # Dependencies for developing OctoPrint plugins @@ -56,6 +59,9 @@ ] ) +# Additional requirements for setup +SETUP_REQUIRES = [] + # Dependency links for any of the aforementioned dependencies DEPENDENCY_LINKS = [] @@ -82,8 +88,22 @@ def params(): version = versioneer.get_version() cmdclass = get_cmdclass() - description = "A responsive web interface for 3D printers" + description = "A snappy web interface for 3D printers" long_description = open("README.md").read() + + install_requires = INSTALL_REQUIRES + extras_require = EXTRA_REQUIRES + dependency_links = DEPENDENCY_LINKS + setup_requires = SETUP_REQUIRES + + try: + import pypandoc + setup_requires += ["setuptools-markdown"] + long_description_markdown_filename = "README.md" + del pypandoc + except: + pass + classifiers = [ "Development Status :: 4 - Beta", "Environment :: Web Environment", @@ -118,9 +138,6 @@ def params(): include_package_data = True zip_safe = False - install_requires = INSTALL_REQUIRES - extras_require = EXTRA_REQUIRES - dependency_links = DEPENDENCY_LINKS if os.environ.get('READTHEDOCS', None) == 'True': # we can't tell read the docs to please perform a pip install -e .[develop], so we help From 2a8e084b51f11b29cfcb389ae74cb1d073831771 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Fri, 23 Oct 2015 13:27:24 +0200 Subject: [PATCH 12/66] Added a missing comma that broke the build... --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e8eb944da..86b7837f9 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,7 @@ # Documentation dependencies "sphinx>=1.3", "sphinxcontrib-httpdomain", - "sphinx_rtd_theme" + "sphinx_rtd_theme", # PyPi upload related "pypandoc" From 91163dc4f400cc614af091980461714e2da9c40a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Thu, 22 Oct 2015 12:34:53 +0200 Subject: [PATCH 13/66] Fixed existing doctests, added some new ones, enabled doctests in nosetests (cherry picked from commit 02c3bf8) --- .travis.yml | 2 +- src/octoprint/filemanager/storage.py | 52 ---------------------------- src/octoprint/printer/profile.py | 4 +-- src/octoprint/util/__init__.py | 51 +++++++++++++++++++-------- 4 files changed, 40 insertions(+), 69 deletions(-) diff --git a/.travis.yml b/.travis.yml index df4eb121f..66a9e7dc1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,4 +6,4 @@ python: install: - pip install -e .[develop] script: -- nosetests tests/ \ No newline at end of file +- nosetests --with-doctest diff --git a/src/octoprint/filemanager/storage.py b/src/octoprint/filemanager/storage.py index 9f238cf93..1fe59e352 100644 --- a/src/octoprint/filemanager/storage.py +++ b/src/octoprint/filemanager/storage.py @@ -584,24 +584,6 @@ def sanitize(self, path): Note that for a ``path`` without a trailing slash the last part will be considered a file name and hence be returned at second position. If you only need to convert a folder path, be sure to include a trailing slash for a string ``path`` or an empty last element for a list ``path``. - - Examples:: - - >>> storage = LocalFileStorage("/some/base/folder") - >>> storage.sanitize("some/folder/and/some file.gco") - ("/some/base/folder/some/folder/and", "some_file.gco") - >>> storage.sanitize(("some", "folder", "and", "some file.gco")) - ("/some/base/folder/some/folder/and", "some_file.gco") - >>> storage.sanitize("some file.gco") - ("/some/base/folder", "some_file.gco") - >>> storage.sanitize(("some file.gco",)) - ("/some/base/folder", "some_file.gco") - >>> storage.sanitize("") - ("/some/base/folder", "") - >>> storage.sanitize("some/folder/with/trailing/slash/") - ("/some/base/folder/some/folder/with/trailing/slash", "") - >>> storage.sanitize("some", "folder", "") - ("/some/base/folder/some/folder", "") """ name = None if isinstance(path, (str, unicode, basestring)): @@ -628,24 +610,6 @@ def sanitize_name(self, name): Raises a :class:`ValueError` for a ``name`` containing ``/`` or ``\``. Otherwise strips any characters from the given ``name`` that are not any of the ASCII characters, digits, ``-``, ``_``, ``.``, ``(``, ``)`` or space and replaces and spaces with ``_``. - - Examples:: - - >>> storage = LocalFileStorage("/some/base/folder") - >>> storage.sanitize_name("some_file.gco") - "some_file.gco" - >>> storage.sanitize_name("some_file with (parentheses) and ümläuts and digits 123.gco") - "some_file_with_(parentheses)_and_mluts_and_digits_123.gco" - >>> storage.sanitize_name("pengüino pequeño.stl") - "pengino_pequeo.stl" - >>> storage.sanitize_name("some/folder/still/left.gco") - Traceback (most recent call last): - File "", line 1, in - ValueError: name must not contain / or \ - >>> storage.sanitize_name("also\\no\\backslashes.gco") - Traceback (most recent call last): - File "", line 1, in - ValueError: name must not contain / or \ """ if name is None: return None @@ -664,22 +628,6 @@ def sanitize_path(self, path): Ensures that the on disk representation of ``path`` is located under the configured basefolder. Resolves all relative path elements (e.g. ``..``) and sanitizes folder names using :func:`sanitize_name`. Final path is the absolute path including leading ``basefolder`` path. - - Examples:: - - >>> storage = LocalFileStorage("/some/base/folder") - >>> storage.sanitize_path("folder/with/subfolder") - "/some/base/folder/folder/with/subfolder" - >>> storage.sanitize_path("folder/with/subfolder/../other/folder") - "/some/base/folder/folder/with/other/folder" - >>> storage.sanitize_path("/folder/with/leading/slash") - "/some/base/folder/folder/with/leading/slash" - >>> storage.sanitize_path(".folder/with/leading/dot") - "/some/base/folder/folder/with/leading/dot - >>> storage.sanitize_path("../../folder/out/of/the/basefolder") - Traceback (most recent call last): - File "", line 1, in - ValueError: path not contained in base folder: /some/folder/out/of/the/basefolder """ if path[0] == "/" or path[0] == ".": path = path[1:] diff --git a/src/octoprint/printer/profile.py b/src/octoprint/printer/profile.py index 88bdfe3e1..7ef630a0b 100644 --- a/src/octoprint/printer/profile.py +++ b/src/octoprint/printer/profile.py @@ -12,7 +12,7 @@ import logging from octoprint.settings import settings -from octoprint.util import dict_merge, dict_clean, dict_contains_keys +from octoprint.util import dict_merge, dict_sanitize, dict_contains_keys class SaveError(Exception): pass @@ -214,7 +214,7 @@ def save(self, profile, allow_overwrite=False, make_default=False): identifier = self._sanitize(identifier) profile["id"] = identifier - profile = dict_clean(profile, self.__class__.default) + profile = dict_sanitize(profile, self.__class__.default) if identifier == "_default": default_profile = dict_merge(self._load_default(), profile) diff --git a/src/octoprint/util/__init__.py b/src/octoprint/util/__init__.py index bc4ddfd71..94b9bb3a8 100644 --- a/src/octoprint/util/__init__.py +++ b/src/octoprint/util/__init__.py @@ -378,6 +378,14 @@ def dict_merge(a, b): Taken from https://www.xormedia.com/recursively-merge-dictionaries-in-python/ + Example:: + + >>> a = dict(foo="foo", bar="bar", fnord=dict(a=1)) + >>> b = dict(foo="other foo", fnord=dict(b=2, l=["some", "list"])) + >>> expected = dict(foo="other foo", bar="bar", fnord=dict(a=1, b=2, l=["some", "list"])) + >>> dict_merge(a, b) == expected + True + Arguments: a (dict): The dictionary to merge ``b`` into b (dict): The dictionary to merge into ``a`` @@ -399,14 +407,24 @@ def dict_merge(a, b): return result -def dict_clean(a, b): +def dict_sanitize(a, b): """ - Recursively deep-cleans ``b`` from ``a``, removing all keys and corresponding values from ``a`` that appear in - ``b``. + Recursively deep-sanitizes ``a`` based on ``b``, removing all keys (and + associated values) from ``a`` that do not appear in ``b``. + + Example:: + + >>> a = dict(foo="foo", bar="bar", fnord=dict(a=1, b=2, l=["some", "list"])) + >>> b = dict(foo=None, fnord=dict(a=None, b=None)) + >>> expected = dict(foo="foo", fnord=dict(a=1, b=2)) + >>> dict_sanitize(a, b) == expected + True + >>> dict_clean(a, b) == expected + True Arguments: - a (dict): The dictionary to clean from ``b``. - b (dict): The dictionary to clean ``b`` from. + a (dict): The dictionary to clean against ``b``. + b (dict): The dictionary containing the key structure to clean from ``a``. Results: dict: A new dict based on ``a`` with all keys (and corresponding values) found in ``b`` removed. @@ -421,21 +439,26 @@ def dict_clean(a, b): if not k in b: del result[k] elif isinstance(v, dict): - result[k] = dict_clean(v, b[k]) + result[k] = dict_sanitize(v, b[k]) else: result[k] = deepcopy(v) return result +dict_clean = deprecated("dict_clean has been renamed to dict_sanitize", + includedoc="Replaced by :func:`dict_sanitize`")(dict_sanitize) -def dict_contains_keys(a, b): +def dict_contains_keys(keys, dictionary): """ - Recursively deep-checks if ``a`` contains all keys found in ``b``. + Recursively deep-checks if ``dictionary`` contains all keys found in ``keys``. Example:: - >>> dict_contains_keys(dict(foo="bar", fnord=dict(a=1, b=2, c=3)), dict(foo="some_other_bar", fnord=dict(b=100))) + >>> positive = dict(foo="some_other_bar", fnord=dict(b=100)) + >>> negative = dict(foo="some_other_bar", fnord=dict(b=100, d=20)) + >>> dictionary = dict(foo="bar", fnord=dict(a=1, b=2, c=3)) + >>> dict_contains_keys(positive, dictionary) True - >>> dict_contains_keys(dict(foo="bar", fnord=dict(a=1, b=2, c=3)), dict(foo="some_other_bar", fnord=dict(b=100, d=20))) + >>> dict_contains_keys(negative, dictionary) False Arguments: @@ -446,14 +469,14 @@ def dict_contains_keys(a, b): boolean: True if all keys found in ``b`` are also present in ``a``, False otherwise. """ - if not isinstance(a, dict) or not isinstance(b, dict): + if not isinstance(keys, dict) or not isinstance(dictionary, dict): return False - for k, v in a.iteritems(): - if not k in b: + for k, v in keys.iteritems(): + if not k in dictionary: return False elif isinstance(v, dict): - if not dict_contains_keys(v, b[k]): + if not dict_contains_keys(v, dictionary[k]): return False return True From 5c8d9007f62277d37cdb470623c335ff718a9dfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Thu, 22 Oct 2015 12:35:23 +0200 Subject: [PATCH 14/66] Fixed some missing exception capturing (cherry picked from commit 2f86a6a) --- src/octoprint/printer/profile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/octoprint/printer/profile.py b/src/octoprint/printer/profile.py index 7ef630a0b..700aef07a 100644 --- a/src/octoprint/printer/profile.py +++ b/src/octoprint/printer/profile.py @@ -408,7 +408,7 @@ def convert_value(profile, path, converter): for path in (("volume", "width"), ("volume", "depth"), ("volume", "height"), ("extruder", "nozzleDiameter")): try: convert_value(profile, path, float) - except: + except Exception as e: self._logger.warn("Profile has invalid value for path {path!r}: {msg}".format(path=".".join(path), msg=str(e))) return False @@ -416,7 +416,7 @@ def convert_value(profile, path, converter): for path in (("axes", "x", "inverted"), ("axes", "y", "inverted"), ("axes", "z", "inverted")): try: convert_value(profile, path, bool) - except: + except Exception as e: self._logger.warn("Profile has invalid value for path {path!r}: {msg}".format(path=".".join(path), msg=str(e))) return False From e223341ece0b4bbc0aa3e0b3f87c562a269cbb93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Thu, 22 Oct 2015 12:40:51 +0200 Subject: [PATCH 15/66] Travis: Migrate from legacy infrastructure http://docs.travis-ci.com/user/migrating-from-legacy/ (cherry picked from commit bec0013) --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 66a9e7dc1..53e5ead92 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,3 +7,4 @@ install: - pip install -e .[develop] script: - nosetests --with-doctest +sudo: false From e8fa2e44e015c9e96484e67422a852d6e7541fe8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Thu, 22 Oct 2015 13:20:11 +0200 Subject: [PATCH 16/66] Travis: Let's see if we can't get the git depth upped a bit (cherry picked from commit 5fc8e22) --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 53e5ead92..dd3b081eb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,3 +8,5 @@ install: script: - nosetests --with-doctest sudo: false +git: + depth: 250 From a5ce07193940ef1c758c669e7ab83129ebbf3165 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Sun, 11 Oct 2015 09:20:15 +0200 Subject: [PATCH 17/66] Make sure only difference to default config is persisted for plugins This change should make sure that only those settings actually make it into config.yaml per plugin that do differ from the default settings. Up until now the full dictionary as received e.g. from the REST API was saved, regardless whether it contained (only) default values or not. That led to a lot of "null" entries in the config.yaml file, which for one looked horrible and also led to problems when a plugin actually wanted to change its default values. With this commit, two things are done. One, upon saving a configuration the default implementation of the save routine in the settings plugin now first diffs the new data against the defaults and only saves what is different. Two, upon server startup a cleanup method is now called on settings plugins whose default implementation fetches the current data stored in config.yaml, diffs that against the defaults and only writes back the diff result. If that is empty the entry is completely deleted from config.yaml. (cherry picked from commit d70fe32) --- src/octoprint/plugin/__init__.py | 28 +++++++--- src/octoprint/plugin/types.py | 78 ++++++++++++++++++++++++--- src/octoprint/server/__init__.py | 11 ++-- src/octoprint/settings.py | 90 ++++++++++++++++++++++++-------- src/octoprint/util/__init__.py | 61 ++++++++++++++++++++++ 5 files changed, 228 insertions(+), 40 deletions(-) diff --git a/src/octoprint/plugin/__init__.py b/src/octoprint/plugin/__init__.py index 21976d08b..84fa12b02 100644 --- a/src/octoprint/plugin/__init__.py +++ b/src/octoprint/plugin/__init__.py @@ -293,19 +293,16 @@ def __init__(self, settings, plugin_key, defaults=None, get_preprocessors=None, self.set_preprocessors = dict(plugins=dict()) self.set_preprocessors["plugins"][plugin_key] = set_preprocessors - def prefix_path(path): - return ['plugins', self.plugin_key] + path - def prefix_path_in_args(args, index=0): result = [] if index == 0: - result.append(prefix_path(args[0])) + result.append(self._prefix_path(args[0])) result.extend(args[1:]) else: args_before = args[:index - 1] args_after = args[index + 1:] result.extend(args_before) - result.append(prefix_path(args[index])) + result.append(self._prefix_path(args[index])) result.extend(args_after) return result @@ -324,6 +321,7 @@ def add_setter_kwargs(kwargs): return kwargs self.access_methods = dict( + has =("has", prefix_path_in_args, add_getter_kwargs), get =("get", prefix_path_in_args, add_getter_kwargs), get_int =("getInt", prefix_path_in_args, add_getter_kwargs), get_float =("getFloat", prefix_path_in_args, add_getter_kwargs), @@ -331,7 +329,8 @@ def add_setter_kwargs(kwargs): set =("set", prefix_path_in_args, add_setter_kwargs), set_int =("setInt", prefix_path_in_args, add_setter_kwargs), set_float =("setFloat", prefix_path_in_args, add_setter_kwargs), - set_boolean=("setBoolean", prefix_path_in_args, add_setter_kwargs) + set_boolean=("setBoolean", prefix_path_in_args, add_setter_kwargs), + remove =("remove", prefix_path_in_args) ) self.deprecated_access_methods = dict( getInt ="get_int", @@ -342,6 +341,17 @@ def add_setter_kwargs(kwargs): setBoolean="set_boolean" ) + def _prefix_path(self, path=None): + if path is None: + path = list() + return ['plugins', self.plugin_key] + path + + def global_has(self, path, **kwargs): + return self.settings.has(path, **kwargs) + + def global_remove(self, path, **kwargs): + return self.settings.remove(path, **kwargs) + def global_get(self, path, **kwargs): """ Getter for retrieving settings not managed by the plugin itself from the core settings structure. Use this @@ -435,6 +445,12 @@ def get_plugin_data_folder(self): os.makedirs(path) return path + def get_all_data(self, **kwargs): + return self.settings.get(self._prefix_path(), **kwargs) + + def clean_all_data(self): + self.settings.remove(self._prefix_path()) + def __getattr__(self, item): all_access_methods = self.access_methods.keys() + self.deprecated_access_methods.keys() if item in all_access_methods: diff --git a/src/octoprint/plugin/types.py b/src/octoprint/plugin/types.py index bf702cb68..d2d95491f 100644 --- a/src/octoprint/plugin/types.py +++ b/src/octoprint/plugin/types.py @@ -764,6 +764,9 @@ def on_after_startup(self): the plugin core system upon initialization of the implementation. """ + config_version_key = "_config_version" + """Key of the field in the settings that holds the configuration format version.""" + def on_settings_load(self): """ Loads the settings for the plugin, called by the Settings API view in order to retrieve all settings from @@ -782,8 +785,8 @@ def on_settings_load(self): :return: the current settings of the plugin, as a dictionary """ data = self._settings.get([], asdict=True, merged=True) - if "_config_version" in data: - del data["_config_version"] + if self.config_version_key in data: + del data[self.config_version_key] return data def on_settings_save(self, data): @@ -795,7 +798,8 @@ def on_settings_save(self, data): .. note:: The default implementation will persist your plugin's settings as is, so just in the structure and in the - types that were received by the Settings API view. + types that were received by the Settings API view. Values identical to the default settings values + will *not* be persisted. If you need more granular control here, e.g. over the used data types, you'll need to override this method and iterate yourself over all your settings, retrieving them (if set) from the supplied received ``data`` @@ -803,15 +807,27 @@ def on_settings_save(self, data): Arguments: data (dict): The settings dictionary to be saved for the plugin + + Returns: + dict: The settings that differed from the defaults and were actually saved. """ import octoprint.util - if "_config_version" in data: - del data["_config_version"] + if self.config_version_key in data: + del data[self.config_version_key] + + # determine diff dict that contains minimal set of changes against the + # default settings - we only want to persist that, not everything + diff = octoprint.util.dict_diff(self.get_settings_defaults(), data) - current = self._settings.get([], asdict=True, merged=True) - merged = octoprint.util.dict_merge(current, data) - self._settings.set([], merged) + version = self.get_settings_version() + + to_persist = dict(diff) + if version: + to_persist[self.config_version_key] = version + self._settings.set([], to_persist) + + return diff def get_settings_defaults(self): """ @@ -894,6 +910,52 @@ def on_settings_migrate(self, target, current): """ pass + def on_settings_cleanup(self): + """ + Called after migration and initialization but before call to :func:`on_settings_initialized`. + + Plugins may overwrite this method to perform additional clean up tasks. + + The default implementation just minimizes the data persisted on disk to only contain + the differences to the defaults (in case the current data was persisted with an older + version of OctoPrint that still duplicated default data). + """ + import octoprint.util + from octoprint.settings import NoSuchSettingsPath + + try: + # let's fetch the current persisted config (so only the data on disk, + # without the defaults) + config = self._settings.get([], asdict=True, incl_defaults=False, error_on_path=True) + except NoSuchSettingsPath: + # no config persisted, nothing to do => get out of here + return + + if config is None: + # config is set to None, that doesn't make sense, kill it and leave + self._settings.clean_all_data() + return + + if self.config_version_key in config and config[self.config_version_key] is None: + # delete None entries for config version - it's the default, no need + del config[self.config_version_key] + + # calculate a minimal diff between the settings and the current config - + # anything already in the settings will be removed from the persisted + # config, no need to duplicate it + defaults = self.get_settings_defaults() + diff = octoprint.util.dict_diff(defaults, config) + + if not diff: + # no diff to defaults, no need to have anything persisted + self._settings.clean_all_data() + else: + # diff => persist only that + self._settings.set([], diff) + + # finally save everything + self._settings.save() + def on_settings_initialized(self): """ Called after the settings have been initialized and - if necessary - also been migrated through a call to diff --git a/src/octoprint/server/__init__.py b/src/octoprint/server/__init__.py index f76f9cb8c..e2a36bac3 100644 --- a/src/octoprint/server/__init__.py +++ b/src/octoprint/server/__init__.py @@ -213,7 +213,7 @@ def settings_plugin_inject_factory(name, implementation): set_preprocessors=set_preprocessors) return dict(settings=plugin_settings) - def settings_plugin_config_migration(name, implementation): + def settings_plugin_config_migration_and_cleanup(name, implementation): if not isinstance(implementation, octoprint.plugin.SettingsPlugin): return @@ -221,12 +221,13 @@ def settings_plugin_config_migration(name, implementation): settings_migrator = implementation.on_settings_migrate if settings_version is not None and settings_migrator is not None: - stored_version = implementation._settings.get_int(["_config_version"]) + stored_version = implementation._settings.get_int([octoprint.plugin.SettingsPlugin.config_version_key]) if stored_version is None or stored_version < settings_version: settings_migrator(settings_version, stored_version) - implementation._settings.set_int(["_config_version"], settings_version) + implementation._settings.set_int([octoprint.plugin.SettingsPlugin.config_version_key], settings_version) implementation._settings.save() + implementation.on_settings_cleanup() implementation.on_settings_initialized() pluginManager.implementation_inject_factories=[octoprint_plugin_inject_factory, settings_plugin_inject_factory] @@ -235,11 +236,11 @@ def settings_plugin_config_migration(name, implementation): settingsPlugins = pluginManager.get_implementations(octoprint.plugin.SettingsPlugin) for implementation in settingsPlugins: try: - settings_plugin_config_migration(implementation._identifier, implementation) + settings_plugin_config_migration_and_cleanup(implementation._identifier, implementation) except: self._logger.exception("Error while trying to migrate settings for plugin {}, ignoring it".format(implementation._identifier)) - pluginManager.implementation_post_inits=[settings_plugin_config_migration] + pluginManager.implementation_post_inits=[settings_plugin_config_migration_and_cleanup] pluginManager.log_all_plugins() diff --git a/src/octoprint/settings.py b/src/octoprint/settings.py index 733d24b8d..122e4413e 100644 --- a/src/octoprint/settings.py +++ b/src/octoprint/settings.py @@ -304,6 +304,11 @@ def settings(init=False, basedir=None, configfile=None): valid_boolean_trues = [True, "true", "yes", "y", "1"] """ Values that are considered to be equivalent to the boolean ``True`` value, used for type conversion in various places.""" + +class NoSuchSettingsPath(BaseException): + pass + + class Settings(object): """ The :class:`Settings` class allows managing all of OctoPrint's settings. It takes care of initializing the settings @@ -810,13 +815,13 @@ def last_modified(self): stat = os.stat(self._configfile) return stat.st_mtime - #~~ getter + ##~~ Internal getter - def get(self, path, asdict=False, config=None, defaults=None, preprocessors=None, merged=False, incl_defaults=True): + def _get_value(self, path, asdict=False, config=None, defaults=None, preprocessors=None, merged=False, incl_defaults=True): import octoprint.util as util if len(path) == 0: - return None + raise NoSuchSettingsPath() if config is None: config = self._config @@ -834,7 +839,7 @@ def get(self, path, asdict=False, config=None, defaults=None, preprocessors=None config = {} defaults = defaults[key] else: - return None + raise NoSuchSettingsPath() if preprocessors and isinstance(preprocessors, dict) and key in preprocessors: preprocessors = preprocessors[key] @@ -858,7 +863,7 @@ def get(self, path, asdict=False, config=None, defaults=None, preprocessors=None elif incl_defaults and key in defaults: value = defaults[key] else: - value = None + raise NoSuchSettingsPath() if preprocessors and isinstance(preprocessors, dict) and key in preprocessors and callable(preprocessors[key]): value = preprocessors[key](value) @@ -876,8 +881,34 @@ def get(self, path, asdict=False, config=None, defaults=None, preprocessors=None else: return results - def getInt(self, path, config=None, defaults=None, preprocessors=None, incl_defaults=True): - value = self.get(path, config=config, defaults=defaults, preprocessors=preprocessors, incl_defaults=incl_defaults) + #~~ has + + def has(self, path, **kwargs): + try: + self._get_value(path, **kwargs) + except NoSuchSettingsPath: + return False + else: + return True + + #~~ getter + + def get(self, path, **kwargs): + error_on_path = kwargs.get("error_on_path", False) + new_kwargs = dict(kwargs) + if "error_on_path" in new_kwargs: + del new_kwargs["error_on_path"] + + try: + return self._get_value(path, **new_kwargs) + except NoSuchSettingsPath: + if error_on_path: + raise + else: + return None + + def getInt(self, path, **kwargs): + value = self.get(path, **kwargs) if value is None: return None @@ -887,8 +918,8 @@ def getInt(self, path, config=None, defaults=None, preprocessors=None, incl_defa self._logger.warn("Could not convert %r to a valid integer when getting option %r" % (value, path)) return None - def getFloat(self, path, config=None, defaults=None, preprocessors=None, incl_defaults=True): - value = self.get(path, config=config, defaults=defaults, preprocessors=preprocessors, incl_defaults=incl_defaults) + def getFloat(self, path, **kwargs): + value = self.get(path, **kwargs) if value is None: return None @@ -898,8 +929,8 @@ def getFloat(self, path, config=None, defaults=None, preprocessors=None, incl_de self._logger.warn("Could not convert %r to a valid integer when getting option %r" % (value, path)) return None - def getBoolean(self, path, config=None, defaults=None, preprocessors=None, incl_defaults=True): - value = self.get(path, config=config, defaults=defaults, preprocessors=preprocessors, incl_defaults=incl_defaults) + def getBoolean(self, path, **kwargs): + value = self.get(path, **kwargs) if value is None: return None if isinstance(value, bool): @@ -952,6 +983,23 @@ def loadScript(self, script_type, name, context=None, source=False): return script + #~~ remove + + def remove(self, path, config=None): + if config is None: + config = self._config + + while len(path) > 1: + key = path.pop(0) + if not isinstance(config, dict) or key not in config: + return + config = config[key] + + key = path.pop(0) + if isinstance(config, dict) and key in config: + del config[key] + self._dirty = True + #~~ setter def set(self, path, value, force=False, defaults=None, config=None, preprocessors=None): @@ -998,9 +1046,9 @@ def set(self, path, value, force=False, defaults=None, config=None, preprocessor config[key] = value self._dirty = True - def setInt(self, path, value, force=False, defaults=None, config=None, preprocessors=None): + def setInt(self, path, value, **kwargs): if value is None: - self.set(path, None, config=config, force=force, defaults=defaults, preprocessors=preprocessors) + self.set(path, None, **kwargs) return try: @@ -1009,11 +1057,11 @@ def setInt(self, path, value, force=False, defaults=None, config=None, preproces self._logger.warn("Could not convert %r to a valid integer when setting option %r" % (value, path)) return - self.set(path, intValue, config=config, force=force, defaults=defaults, preprocessors=preprocessors) + self.set(path, intValue, **kwargs) - def setFloat(self, path, value, force=False, defaults=None, config=None, preprocessors=None): + def setFloat(self, path, value, **kwargs): if value is None: - self.set(path, None, config=config, force=force, defaults=defaults, preprocessors=preprocessors) + self.set(path, None, **kwargs) return try: @@ -1022,15 +1070,15 @@ def setFloat(self, path, value, force=False, defaults=None, config=None, preproc self._logger.warn("Could not convert %r to a valid integer when setting option %r" % (value, path)) return - self.set(path, floatValue, config=config, force=force, defaults=defaults, preprocessors=preprocessors) + self.set(path, floatValue, **kwargs) - def setBoolean(self, path, value, force=False, defaults=None, config=None, preprocessors=None): + def setBoolean(self, path, value, **kwargs): if value is None or isinstance(value, bool): - self.set(path, value, config=config, force=force, defaults=defaults, preprocessors=preprocessors) + self.set(path, value, **kwargs) elif value.lower() in valid_boolean_trues: - self.set(path, True, config=config, force=force, defaults=defaults, preprocessors=preprocessors) + self.set(path, True, **kwargs) else: - self.set(path, False, config=config, force=force, defaults=defaults, preprocessors=preprocessors) + self.set(path, False, **kwargs) def setBaseFolder(self, type, path, force=False): if type not in default_settings["folder"].keys(): diff --git a/src/octoprint/util/__init__.py b/src/octoprint/util/__init__.py index 94b9bb3a8..33459a501 100644 --- a/src/octoprint/util/__init__.py +++ b/src/octoprint/util/__init__.py @@ -447,6 +447,67 @@ def dict_sanitize(a, b): includedoc="Replaced by :func:`dict_sanitize`")(dict_sanitize) +def dict_diff(a, b): + """ + Recursively calculates the minimal dict that would be needed to be deep merged with + a in order to produce the same result as deep merging a and b. + + Example:: + + >>> a = dict(foo=dict(a=1, b=2), bar=dict(c=3, d=4)) + >>> b = dict(bar=dict(c=3, d=5), fnord=None) + >>> c = dict_diff(a, b) + >>> c == dict(bar=dict(d=5), fnord=None) + True + >>> dict_merge(a, c) == dict_merge(a, b) + True + + Arguments: + a (dict): Source dictionary + b (dict): Dictionary to compare to source dictionary and derive diff for + + Returns: + dict: The minimal dictionary to deep merge on a to get the same result + as deep merging b on a. + """ + + if not isinstance(a, dict) or not isinstance(b, dict): + raise ValueError("a and b must be dictionaries") + + if a == b: + # shortcut: if both are equal, we return an empty dict as result + return dict() + + from copy import deepcopy + + all_keys = set(a.keys() + b.keys()) + result = dict() + for k in all_keys: + if k not in b: + # key not contained in b => not contained in result + continue + + if k in a: + # key is present in both dicts, we have to take a look at the value + value_a = a[k] + value_b = b[k] + + if value_a != value_b: + # we only need to look further if the values are not equal + + if isinstance(value_a, dict) and isinstance(value_b, dict): + # both are dicts => deeper down it goes into the rabbit hole + result[k] = dict_diff(value_a, value_b) + else: + # new b wins over old a + result[k] = deepcopy(value_b) + + else: + # key is new, add it + result[k] = deepcopy(b[k]) + return result + + def dict_contains_keys(keys, dictionary): """ Recursively deep-checks if ``dictionary`` contains all keys found in ``keys``. From 9b3ef8b693c2b083376ac3590574373544c1fa5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Tue, 20 Oct 2015 10:35:47 +0200 Subject: [PATCH 18/66] Fix: Persist all data different from default, not just current changes We need to merge with our current data since we only might get partial data from our caller. (cherry picked from commit 7ea1578) --- src/octoprint/plugin/__init__.py | 12 ++++++++++++ src/octoprint/plugin/types.py | 11 ++++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/octoprint/plugin/__init__.py b/src/octoprint/plugin/__init__.py index 84fa12b02..0cc587150 100644 --- a/src/octoprint/plugin/__init__.py +++ b/src/octoprint/plugin/__init__.py @@ -446,6 +446,18 @@ def get_plugin_data_folder(self): return path def get_all_data(self, **kwargs): + merged = kwargs.get("merged", True) + asdict = kwargs.get("asdict", True) + defaults = kwargs.get("defaults", self.defaults) + preprocessors = kwargs.get("preprocessors", self.get_preprocessors) + + kwargs.update(dict( + merged=merged, + asdict=asdict, + defaults=defaults, + preprocessors=preprocessors + )) + return self.settings.get(self._prefix_path(), **kwargs) def clean_all_data(self): diff --git a/src/octoprint/plugin/types.py b/src/octoprint/plugin/types.py index d2d95491f..1b67436b5 100644 --- a/src/octoprint/plugin/types.py +++ b/src/octoprint/plugin/types.py @@ -813,12 +813,17 @@ def on_settings_save(self, data): """ import octoprint.util - if self.config_version_key in data: - del data[self.config_version_key] + # get the current data + current = self._settings.get_all_data() + + # merge our new data on top of it + new_current = octoprint.util.dict_merge(current, data) + if self.config_version_key in new_current: + del new_current[self.config_version_key] # determine diff dict that contains minimal set of changes against the # default settings - we only want to persist that, not everything - diff = octoprint.util.dict_diff(self.get_settings_defaults(), data) + diff = octoprint.util.dict_diff(self.get_settings_defaults(), new_current) version = self.get_settings_version() From 063251fe03b51c1a40659497c34263c21735d57d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Tue, 20 Oct 2015 15:41:44 +0200 Subject: [PATCH 19/66] Use get_all_data instead of get([], ...) construct (cherry picked from commit ffcbdba) --- src/octoprint/plugin/types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/octoprint/plugin/types.py b/src/octoprint/plugin/types.py index 1b67436b5..da142d45b 100644 --- a/src/octoprint/plugin/types.py +++ b/src/octoprint/plugin/types.py @@ -931,7 +931,7 @@ def on_settings_cleanup(self): try: # let's fetch the current persisted config (so only the data on disk, # without the defaults) - config = self._settings.get([], asdict=True, incl_defaults=False, error_on_path=True) + config = self._settings.get_all_data(merged=False, incl_defaults=False, error_on_path=True) except NoSuchSettingsPath: # no config persisted, nothing to do => get out of here return From b4cb0dec87fc7f30e4cdea64f362b0b4a67b92d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Tue, 20 Oct 2015 17:57:05 +0200 Subject: [PATCH 20/66] Unit tests and some fixes for improved plugin settings processing (cherry picked from commit bcd0f34) --- src/octoprint/plugin/types.py | 13 +- src/octoprint/server/__init__.py | 3 +- tests/plugin/test_types_settings.py | 313 ++++++++++++++++++++++++++++ 3 files changed, 323 insertions(+), 6 deletions(-) create mode 100644 tests/plugin/test_types_settings.py diff --git a/src/octoprint/plugin/types.py b/src/octoprint/plugin/types.py index da142d45b..6fada118b 100644 --- a/src/octoprint/plugin/types.py +++ b/src/octoprint/plugin/types.py @@ -784,7 +784,7 @@ def on_settings_load(self): :return: the current settings of the plugin, as a dictionary """ - data = self._settings.get([], asdict=True, merged=True) + data = self._settings.get_all_data() if self.config_version_key in data: del data[self.config_version_key] return data @@ -815,6 +815,8 @@ def on_settings_save(self, data): # get the current data current = self._settings.get_all_data() + if current is None: + current = dict() # merge our new data on top of it new_current = octoprint.util.dict_merge(current, data) @@ -830,7 +832,11 @@ def on_settings_save(self, data): to_persist = dict(diff) if version: to_persist[self.config_version_key] = version - self._settings.set([], to_persist) + + if to_persist: + self._settings.set([], to_persist) + else: + self._settings.clean_all_data() return diff @@ -958,9 +964,6 @@ def on_settings_cleanup(self): # diff => persist only that self._settings.set([], diff) - # finally save everything - self._settings.save() - def on_settings_initialized(self): """ Called after the settings have been initialized and - if necessary - also been migrated through a call to diff --git a/src/octoprint/server/__init__.py b/src/octoprint/server/__init__.py index e2a36bac3..4c46f67db 100644 --- a/src/octoprint/server/__init__.py +++ b/src/octoprint/server/__init__.py @@ -225,9 +225,10 @@ def settings_plugin_config_migration_and_cleanup(name, implementation): if stored_version is None or stored_version < settings_version: settings_migrator(settings_version, stored_version) implementation._settings.set_int([octoprint.plugin.SettingsPlugin.config_version_key], settings_version) - implementation._settings.save() implementation.on_settings_cleanup() + implementation._settings.save() + implementation.on_settings_initialized() pluginManager.implementation_inject_factories=[octoprint_plugin_inject_factory, settings_plugin_inject_factory] diff --git a/tests/plugin/test_types_settings.py b/tests/plugin/test_types_settings.py new file mode 100644 index 000000000..31a53df7b --- /dev/null +++ b/tests/plugin/test_types_settings.py @@ -0,0 +1,313 @@ +import unittest +import mock + +import octoprint.plugin + +class TestSettingsPlugin(unittest.TestCase): + + def setUp(self): + self.settings = mock.MagicMock() + + self.plugin = octoprint.plugin.SettingsPlugin() + self.plugin._settings = self.settings + + def test_on_settings_cleanup(self): + """Tests that after cleanup only minimal config is left in storage.""" + + ### setup + + # settings defaults + defaults = dict( + foo=dict( + a=1, + b=2, + l1=["some", "list"], + l2=["another", "list"] + ), + bar=True, + fnord=None + ) + self.plugin.get_settings_defaults = mock.MagicMock() + self.plugin.get_settings_defaults.return_value = defaults + + # stored config, containing one redundant entry (bar=True, same as default) + in_config = dict( + foo=dict( + l1=["some", "other", "list"], + l2=["another", "list"], + l3=["a", "third", "list"] + ), + bar=True, + fnord=dict( + c=3, + d=4 + ) + ) + self.settings.get_all_data.return_value = in_config + + ### execute + + self.plugin.on_settings_cleanup() + + ### assert + + # minimal config (current without redundant value) should have been set + expected = dict( + foo=dict( + l1=["some", "other", "list"], + l3=["a", "third", "list"] + ), + fnord=dict( + c=3, + d=4 + ) + ) + self.settings.set.assert_called_once_with([], expected) + + def test_on_settings_cleanup_configversion(self): + """Tests that set config version is always left stored.""" + + ### setup + + defaults = dict( + foo="fnord" + ) + self.plugin.get_settings_defaults = mock.MagicMock() + self.plugin.get_settings_defaults.return_value = defaults + + in_config = dict( + _config_version=1, + foo="fnord" + ) + self.settings.get_all_data.return_value = in_config + + ### execute + + self.plugin.on_settings_cleanup() + + ### assert + + # minimal config incl. config version should have been set + self.settings.set.assert_called_once_with([], dict(_config_version=1)) + + def test_on_settings_cleanup_noconfigversion(self): + """Tests that config versions of None are cleaned from stored data.""" + + ### setup + + defaults = dict( + foo="bar" + ) + self.plugin.get_settings_defaults = mock.MagicMock() + self.plugin.get_settings_defaults.return_value = defaults + + # stored config version is None + in_config = dict( + _config_version=None, + foo="fnord" + ) + self.settings.get_all_data.return_value = in_config + + ### execute + + self.plugin.on_settings_cleanup() + + ### assert + + # minimal config without config version should have been set + self.settings.set.assert_called_once_with([], dict(foo="fnord")) + + def test_on_settings_cleanup_emptydiff(self): + """Tests that settings are cleaned up if the diff data <-> defaults is empty.""" + + ### setup + + defaults = dict( + foo="bar" + ) + self.plugin.get_settings_defaults = mock.MagicMock() + self.plugin.get_settings_defaults.return_value = defaults + + # current stored config, same as defaults + in_config = dict( + foo="bar" + ) + self.settings.get_all_data.return_value = in_config + + ### execute + + self.plugin.on_settings_cleanup() + + ### assert + + # should have been cleared + self.settings.clean_all_data.assert_called_once_with() + + def test_on_settings_cleanup_nosuchpath(self): + """Tests that no processing is done if nothing is stored in settings.""" + + from octoprint.settings import NoSuchSettingsPath + + ### setup + + # simulate no settings stored in config.yaml + self.settings.get_all_data.side_effect = NoSuchSettingsPath() + + ### execute + + self.plugin.on_settings_cleanup() + + ### assert + + # only get_all_data should have been called + self.settings.get_all_data.assert_called_once_with(merged=False, incl_defaults=False, error_on_path=True) + self.assertTrue(len(self.settings.method_calls) == 1) + + def test_on_settings_cleanup_none(self): + """Tests the None entries in config get cleaned up.""" + + ### setup + + # simulate None entry in config.yaml + self.settings.get_all_data.return_value = None + + ### execute + + self.plugin.on_settings_cleanup() + + ### assert + + # should have been cleaned + self.settings.clean_all_data.assert_called_once_with() + + def test_on_settings_save(self): + """Tests that only the diff is saved.""" + + ### setup + + current = dict( + foo="bar" + ) + self.settings.get_all_data.return_value = current + + defaults = dict( + foo="foo", + bar=dict( + a=1, + b=2 + ) + ) + self.plugin.get_settings_defaults = mock.MagicMock() + self.plugin.get_settings_defaults.return_value = defaults + + ### execute + + data = dict( + foo="fnord", + bar=dict( + a=1, + b=2 + ) + ) + diff = self.plugin.on_settings_save(data) + + ### assert + + # the minimal diff should have been saved + expected = dict( + foo="fnord" + ) + self.settings.set.assert_called_once_with([], expected) + + self.assertEquals(diff, expected) + + def test_on_settings_save_nodiff(self): + """Tests that data is cleaned if there's not difference between data and defaults.""" + + ### setup + + self.settings.get_all_data.return_value = None + + defaults = dict( + foo="bar", + bar=dict( + a=1, + b=2, + l=["some", "list"] + ) + ) + self.plugin.get_settings_defaults = mock.MagicMock() + self.plugin.get_settings_defaults.return_value = defaults + + ### execute + + data = dict(foo="bar") + diff = self.plugin.on_settings_save(data) + + ### assert + + self.settings.clean_all_data.assert_called_once_with() + self.assertEquals(diff, dict()) + + def test_on_settings_save_configversion(self): + """Tests that saved data gets stripped config version and set correct one.""" + + ### setup + + self.settings.get_all_data.return_value = None + + defaults = dict( + foo="bar" + ) + self.plugin.get_settings_defaults = mock.MagicMock() + self.plugin.get_settings_defaults.return_value = defaults + + version = 1 + self.plugin.get_settings_version = mock.MagicMock() + self.plugin.get_settings_version.return_value = version + + ### execute + + data = dict(_config_version=None, foo="bar") + diff = self.plugin.on_settings_save(data) + + ### assert + + expected_diff = dict() + expected_set = dict(_config_version=version) + + # while there was no diff, we should still have saved the new config version + self.settings.set.assert_called_once_with([], expected_set) + + self.assertEquals(diff, expected_diff) + + def test_on_settings_load(self): + """Tests that on_settings_load returns what's stored in the config, without config version.""" + + ### setup + + # current data incl. config version + current = dict( + _config_version=3, + foo="bar", + fnord=dict( + a=1, + b=2, + l=["some", "list"] + ) + ) + + # expected is current without _config_version - we make the copy now + # since our current dict will be modified by the test + expected = dict(current) + del expected["_config_version"] + + self.settings.get_all_data.return_value = expected + + ### execute + + result = self.plugin.on_settings_load() + + ### assert + + self.assertEquals(result, expected) From 33306b6787d15ef1d128a5fcfc3b727f212218e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Mon, 26 Oct 2015 09:11:44 +0100 Subject: [PATCH 21/66] Fixed an earlier merge error --- src/octoprint/plugin/types.py | 4 ++-- src/octoprint/util/__init__.py | 40 +++++++++++++++++----------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/octoprint/plugin/types.py b/src/octoprint/plugin/types.py index 6fada118b..7f76126c2 100644 --- a/src/octoprint/plugin/types.py +++ b/src/octoprint/plugin/types.py @@ -825,7 +825,7 @@ def on_settings_save(self, data): # determine diff dict that contains minimal set of changes against the # default settings - we only want to persist that, not everything - diff = octoprint.util.dict_diff(self.get_settings_defaults(), new_current) + diff = octoprint.util.dict_minimal_mergediff(self.get_settings_defaults(), new_current) version = self.get_settings_version() @@ -955,7 +955,7 @@ def on_settings_cleanup(self): # anything already in the settings will be removed from the persisted # config, no need to duplicate it defaults = self.get_settings_defaults() - diff = octoprint.util.dict_diff(defaults, config) + diff = octoprint.util.dict_minimal_mergediff(defaults, config) if not diff: # no diff to defaults, no need to have anything persisted diff --git a/src/octoprint/util/__init__.py b/src/octoprint/util/__init__.py index 33459a501..61e200761 100644 --- a/src/octoprint/util/__init__.py +++ b/src/octoprint/util/__init__.py @@ -447,7 +447,7 @@ def dict_sanitize(a, b): includedoc="Replaced by :func:`dict_sanitize`")(dict_sanitize) -def dict_diff(a, b): +def dict_minimal_mergediff(source, target): """ Recursively calculates the minimal dict that would be needed to be deep merged with a in order to produce the same result as deep merging a and b. @@ -456,55 +456,55 @@ def dict_diff(a, b): >>> a = dict(foo=dict(a=1, b=2), bar=dict(c=3, d=4)) >>> b = dict(bar=dict(c=3, d=5), fnord=None) - >>> c = dict_diff(a, b) + >>> c = dict_minimal_mergediff(a, b) >>> c == dict(bar=dict(d=5), fnord=None) True >>> dict_merge(a, c) == dict_merge(a, b) True Arguments: - a (dict): Source dictionary - b (dict): Dictionary to compare to source dictionary and derive diff for + source (dict): Source dictionary + target (dict): Dictionary to compare to source dictionary and derive diff for Returns: - dict: The minimal dictionary to deep merge on a to get the same result - as deep merging b on a. + dict: The minimal dictionary to deep merge on ``source`` to get the same result + as deep merging ``target`` on ``source``. """ - if not isinstance(a, dict) or not isinstance(b, dict): - raise ValueError("a and b must be dictionaries") + if not isinstance(source, dict) or not isinstance(target, dict): + raise ValueError("source and target must be dictionaries") - if a == b: + if source == target: # shortcut: if both are equal, we return an empty dict as result return dict() from copy import deepcopy - all_keys = set(a.keys() + b.keys()) + all_keys = set(source.keys() + target.keys()) result = dict() for k in all_keys: - if k not in b: - # key not contained in b => not contained in result + if k not in target: + # key not contained in target => not contained in result continue - if k in a: + if k in source: # key is present in both dicts, we have to take a look at the value - value_a = a[k] - value_b = b[k] + value_source = source[k] + value_target = target[k] - if value_a != value_b: + if value_source != value_target: # we only need to look further if the values are not equal - if isinstance(value_a, dict) and isinstance(value_b, dict): + if isinstance(value_source, dict) and isinstance(value_target, dict): # both are dicts => deeper down it goes into the rabbit hole - result[k] = dict_diff(value_a, value_b) + result[k] = dict_minimal_mergediff(value_source, value_target) else: # new b wins over old a - result[k] = deepcopy(value_b) + result[k] = deepcopy(value_target) else: # key is new, add it - result[k] = deepcopy(b[k]) + result[k] = deepcopy(target[k]) return result From 084fd8d4336ec69583600e9f6ba30b29982baed4 Mon Sep 17 00:00:00 2001 From: Marcel Hellwig <1hellwig@informatik.uni-hamburg.de> Date: Fri, 11 Sep 2015 11:20:41 +0200 Subject: [PATCH 22/66] add errorhandler decorator to blueprintplugin This will allow you to add an errorhandler for your blueprint easily. (cherry picked from commit b6053c1) --- src/octoprint/plugin/types.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/octoprint/plugin/types.py b/src/octoprint/plugin/types.py index 7f76126c2..0d54a9801 100644 --- a/src/octoprint/plugin/types.py +++ b/src/octoprint/plugin/types.py @@ -651,6 +651,24 @@ def decorator(f): return f return decorator + @staticmethod + def errorhandler(code_or_exception): + """ + A decorator to mark errorhandlings methods in your BlueprintPlugin subclass. Works just the same as Flask's + own ``errorhandler`` decorator available on blueprints. + + See `the documentation for flask.Blueprint.errorhandler `_ + and `the documentation for flask.Flask.errorhandler `_ for more + information. + """ + from collections import defaultdict + def decorator(f): + if not hasattr(f, "_blueprint_error_handler") or f._blueprint_error_handler is None: + f._blueprint_error_handler = defaultdict(list) + f._blueprint_error_handler[f.__name__].append(code_or_exception) + return f + return decorator + def get_blueprint(self): """ Creates and returns the blueprint for your plugin. Override this if you want to define and handle your blueprint yourself. @@ -669,6 +687,9 @@ def get_blueprint(self): for blueprint_rule in f._blueprint_rules[member]: rule, options = blueprint_rule blueprint.add_url_rule(rule, options.pop("endpoint", f.__name__), view_func=f, **options) + if hasattr(f, "_blueprint_error_handler") and member in f._blueprint_error_handler: + for code_or_exception in f._blueprint_error_handler[member]: + blueprint.errorhandler(code_or_exception)(f) return blueprint def get_blueprint_kwargs(self): From a53195d2a0b5b5007f2730eda6275fe947289d26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Thu, 22 Oct 2015 11:07:55 +0200 Subject: [PATCH 23/66] Added Unit Tests for BlueprintPlugin mixin (cherry picked from commit 53a62ab) --- src/octoprint/plugin/types.py | 16 +++ tests/plugin/test_types_blueprint.py | 139 +++++++++++++++++++++++++++ 2 files changed, 155 insertions(+) create mode 100644 tests/plugin/test_types_blueprint.py diff --git a/src/octoprint/plugin/types.py b/src/octoprint/plugin/types.py index 0d54a9801..d1987ea6e 100644 --- a/src/octoprint/plugin/types.py +++ b/src/octoprint/plugin/types.py @@ -678,18 +678,34 @@ def get_blueprint(self): :return: the blueprint ready to be registered with Flask """ + if hasattr(self, "_blueprint"): + # if we already constructed the blueprint and hence have it cached, + # return that instance - we don't want to instance it multiple times + return self._blueprint + import flask kwargs = self.get_blueprint_kwargs() blueprint = flask.Blueprint("plugin." + self._identifier, self._identifier, **kwargs) + + # we now iterate over all members of ourselves and look if we find an attribute + # that has data originating from one of our decorators - we ignore anything + # starting with a _ to only handle public stuff for member in [member for member in dir(self) if not member.startswith("_")]: f = getattr(self, member) + if hasattr(f, "_blueprint_rules") and member in f._blueprint_rules: + # this attribute was annotated with our @route decorator for blueprint_rule in f._blueprint_rules[member]: rule, options = blueprint_rule blueprint.add_url_rule(rule, options.pop("endpoint", f.__name__), view_func=f, **options) + if hasattr(f, "_blueprint_error_handler") and member in f._blueprint_error_handler: + # this attribute was annotated with our @error_handler decorator for code_or_exception in f._blueprint_error_handler[member]: blueprint.errorhandler(code_or_exception)(f) + + # cache and return the blueprint object + self._blueprint = blueprint return blueprint def get_blueprint_kwargs(self): diff --git a/tests/plugin/test_types_blueprint.py b/tests/plugin/test_types_blueprint.py new file mode 100644 index 000000000..17df406c3 --- /dev/null +++ b/tests/plugin/test_types_blueprint.py @@ -0,0 +1,139 @@ +import unittest +import mock + +import octoprint.plugin + +class BlueprintPluginTest(unittest.TestCase): + + def setUp(self): + self.basefolder = "/some/funny/basefolder" + + self.plugin = octoprint.plugin.BlueprintPlugin() + self.plugin._basefolder = self.basefolder + + class MyAssetPlugin(octoprint.plugin.BlueprintPlugin, octoprint.plugin.AssetPlugin): + def get_asset_folder(self): + return "/some/asset/folder" + + class MyTemplatePlugin(octoprint.plugin.BlueprintPlugin, octoprint.plugin.TemplatePlugin): + def get_template_folder(self): + return "/some/template/folder" + + self.assetplugin = MyAssetPlugin() + self.assetplugin._basefolder = self.basefolder + + self.templateplugin = MyTemplatePlugin() + self.templateplugin._basefolder = self.basefolder + + def test_route(self): + + def test_method(): + pass + + octoprint.plugin.BlueprintPlugin.route("/test/method", methods=["GET"])(test_method) + octoprint.plugin.BlueprintPlugin.route("/test/method/{foo}", methods=["PUT"])(test_method) + + self.assertTrue(hasattr(test_method, "_blueprint_rules")) + self.assertTrue("test_method" in test_method._blueprint_rules) + self.assertTrue(len(test_method._blueprint_rules["test_method"]) == 2) + self.assertListEqual(test_method._blueprint_rules["test_method"], [ + ("/test/method", dict(methods=["GET"])), + ("/test/method/{foo}", dict(methods=["PUT"])) + ]) + + def test_errorhandler(self): + + def test_method(): + pass + + octoprint.plugin.BlueprintPlugin.errorhandler(404)(test_method) + + self.assertTrue(hasattr(test_method, "_blueprint_error_handler")) + self.assertTrue("test_method" in test_method._blueprint_error_handler) + self.assertTrue(len(test_method._blueprint_error_handler["test_method"]) == 1) + self.assertListEqual(test_method._blueprint_error_handler["test_method"], [ + 404 + ]) + + def test_get_blueprint_kwargs(self): + import os + expected = dict( + static_folder=os.path.join(self.basefolder, "static"), + template_folder=os.path.join(self.basefolder, "templates") + ) + + result = self.plugin.get_blueprint_kwargs() + + self.assertEquals(result, expected) + + def test_get_blueprint_kwargs_assetplugin(self): + import os + expected = dict( + static_folder=self.assetplugin.get_asset_folder(), + template_folder=os.path.join(self.basefolder, "templates") + ) + + result = self.assetplugin.get_blueprint_kwargs() + + self.assertEquals(result, expected) + + def test_get_blueprint_kwargs_templateplugin(self): + import os + expected = dict( + static_folder=os.path.join(self.basefolder, "static"), + template_folder=self.templateplugin.get_template_folder() + ) + + result = self.templateplugin.get_blueprint_kwargs() + + self.assertEquals(result, expected) + + def test_get_blueprint(self): + import os + expected_kwargs = dict( + static_folder=os.path.join(self.basefolder, "static"), + template_folder=os.path.join(self.basefolder, "templates") + ) + + class MyPlugin(octoprint.plugin.BlueprintPlugin): + + @octoprint.plugin.BlueprintPlugin.route("/some/path", methods=["GET"]) + def route_method(self): + pass + + @octoprint.plugin.BlueprintPlugin.errorhandler(404) + def errorhandler_method(self): + pass + + @octoprint.plugin.BlueprintPlugin.route("/hidden/path", methods=["GET"]) + def _hidden_method(self): + pass + + plugin = MyPlugin() + plugin._basefolder = self.basefolder + plugin._identifier = "myplugin" + + with mock.patch("flask.Blueprint") as MockBlueprint: + blueprint = mock.MagicMock() + MockBlueprint.return_value = blueprint + + errorhandler = mock.MagicMock() + blueprint.errorhandler.return_value = errorhandler + + result = plugin.get_blueprint() + + self.assertEquals(result, blueprint) + + MockBlueprint.assert_called_once_with("plugin.myplugin", "myplugin", **expected_kwargs) + blueprint.add_url_rule.assert_called_once_with("/some/path", "route_method", view_func=plugin.route_method, methods=["GET"]) + + blueprint.errorhandler.assert_called_once_with(404) + errorhandler.assert_called_once_with(plugin.errorhandler_method) + + def test_get_blueprint_cached(self): + blueprint = mock.MagicMock() + self.plugin._blueprint = blueprint + + result = self.plugin.get_blueprint() + + self.assertEquals(blueprint, result) From 37730616c1acfb6620741e1fb684a385518a0c7f Mon Sep 17 00:00:00 2001 From: Andres Date: Sun, 20 Sep 2015 22:44:49 +0200 Subject: [PATCH 24/66] Fixed wrong urls (cherry picked from commit 119f3ba) --- docs/api/job.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/api/job.rst b/docs/api/job.rst index 3bcc1fde2..eeb3b0718 100644 --- a/docs/api/job.rst +++ b/docs/api/job.rst @@ -37,7 +37,7 @@ Issue a job command .. sourcecode:: http - POST /api/control/job HTTP/1.1 + POST /api/job HTTP/1.1 Host: example.com Content-Type: application/json X-Api-Key: abcdef... @@ -54,7 +54,7 @@ Issue a job command .. sourcecode:: http - POST /api/control/job HTTP/1.1 + POST /api/job HTTP/1.1 Host: example.com Content-Type: application/json X-Api-Key: abcdef... @@ -71,7 +71,7 @@ Issue a job command .. sourcecode:: http - POST /api/control/job HTTP/1.1 + POST /api/job HTTP/1.1 Host: example.com Content-Type: application/json X-Api-Key: abcdef... @@ -88,7 +88,7 @@ Issue a job command .. sourcecode:: http - POST /api/control/job HTTP/1.1 + POST /api/job HTTP/1.1 Host: example.com Content-Type: application/json X-Api-Key: abcdef... From 65efcec0081c2d258bc95dc400f36cad085beadd Mon Sep 17 00:00:00 2001 From: Kevin Murphy Date: Fri, 30 Oct 2015 11:18:44 -0400 Subject: [PATCH 25/66] Change instance of row to row-fluid to avoid conflict with responsive CSS --- src/octoprint/templates/tabs/temperature.jinja2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/octoprint/templates/tabs/temperature.jinja2 b/src/octoprint/templates/tabs/temperature.jinja2 index 90622dcdd..103443428 100644 --- a/src/octoprint/templates/tabs/temperature.jinja2 +++ b/src/octoprint/templates/tabs/temperature.jinja2 @@ -1,5 +1,5 @@ {% if enableTemperatureGraph %} -
+
{% endif %} From a44eb43f0f37fdd28d8119377f99ed25bd9b835e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Fri, 30 Oct 2015 16:53:17 +0100 Subject: [PATCH 26/66] Added @kevingelion to AUTHORS.md --- AUTHORS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS.md b/AUTHORS.md index 0309fdf53..3cd1754d2 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -53,6 +53,7 @@ date of first contribution): * [Andrew Erickson](https://github.com/aerickson) * [Nicanor Romero Venier](https://github.com/nicanor-romero) * [Thomas Hou](https://github.com/masterhou) + * [Kevin Murphy](https://github.com/kevingelion) OctoPrint started off as a fork of [Cura](https://github.com/daid/Cura) by [Daid Braam](https://github.com/daid). Parts of its communication layer and From 6b0dee26e53089c6543614c7152e2d4c34869a5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Thu, 5 Nov 2015 17:33:27 +0100 Subject: [PATCH 27/66] Interpret M25 as a pause when not printing from SD too --- src/octoprint/util/comm.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/octoprint/util/comm.py b/src/octoprint/util/comm.py index 9bc2c7024..a0308fda4 100644 --- a/src/octoprint/util/comm.py +++ b/src/octoprint/util/comm.py @@ -1701,6 +1701,12 @@ def _gcode_M0_queuing(self, cmd, cmd_type=None): return None, # Don't send the M0 or M1 to the machine, as M0 and M1 are handled as an LCD menu pause. _gcode_M1_queuing = _gcode_M0_queuing + def _gcode_M25_queuing(self, cmd, cmd_type=None): + # M25 while not printing from SD will be handled as pause. This way it can be used as another marker + # for GCODE induced pausing. Send it to the printer anyway though. + if self.isPrinting() and not self.isSdPrinting(): + self.setPause(True) + def _gcode_M104_sent(self, cmd, cmd_type=None): toolNum = self._currentTool toolMatch = regexes_parameters["intT"].search(cmd) From b98348de029003bc0333ccfb2ff2e071d1a74c8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Fri, 6 Nov 2015 11:46:12 +0100 Subject: [PATCH 28/66] Cache rendered page and translation files indefinitely --- src/octoprint/server/util/flask.py | 56 ++++++++++++++++++++++++++++-- src/octoprint/server/views.py | 7 ++-- 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/src/octoprint/server/util/flask.py b/src/octoprint/server/util/flask.py index 7fe8ae34a..f4d7a9d9b 100644 --- a/src/octoprint/server/util/flask.py +++ b/src/octoprint/server/util/flask.py @@ -26,7 +26,7 @@ import octoprint.users import octoprint.plugin -from werkzeug.contrib.cache import SimpleCache +from werkzeug.contrib.cache import BaseCache #~~ monkey patching @@ -259,7 +259,59 @@ def passive_login(): #~~ cache decorator for cacheable views -_cache = SimpleCache() +class LessSimpleCache(BaseCache): + """ + Slightly improved version of :class:`SimpleCache`. + + Setting ``default_timeout`` or ``timeout`` to ``-1`` will have no timeout be applied at all. + """ + + def __init__(self, threshold=500, default_timeout=300): + BaseCache.__init__(self, default_timeout=default_timeout) + self._cache = {} + self.clear = self._cache.clear + self._threshold = threshold + + def _prune(self): + if self.over_threshold(): + now = time.time() + for idx, (key, (expires, _)) in enumerate(self._cache.items()): + if expires is not None and expires <= now or idx % 3 == 0: + self._cache.pop(key, None) + + def get(self, key): + import pickle + now = time.time() + expires, value = self._cache.get(key, (0, None)) + if expires is None or expires > now: + return pickle.loads(value) + + def set(self, key, value, timeout=None): + import pickle + self._prune() + self._cache[key] = (self.calculate_timeout(timeout=timeout), + pickle.dumps(value, pickle.HIGHEST_PROTOCOL)) + + def add(self, key, value, timeout=None): + self.set(key, value, timeout=None) + self._cache.setdefault(key, self._cache[key]) + + def delete(self, key): + self._cache.pop(key, None) + + def calculate_timeout(self, timeout=None): + if timeout is None: + timeout = self.default_timeout + if timeout is -1: + return None + return time.time() + timeout + + def over_threshold(self): + if self._threshold is None: + return False + return len(self._cache) > self._threshold + +_cache = LessSimpleCache() def cached(timeout=5 * 60, key=lambda: "view/%s" % flask.request.path, unless=None, refreshif=None, unless_response=None): def decorator(f): diff --git a/src/octoprint/server/views.py b/src/octoprint/server/views.py index f89bd9dad..179275368 100644 --- a/src/octoprint/server/views.py +++ b/src/octoprint/server/views.py @@ -27,7 +27,8 @@ _valid_div_re = re.compile("[a-zA-Z_-]+") @app.route("/") -@util.flask.cached(refreshif=lambda: util.flask.cache_check_headers() or "_refresh" in request.values, +@util.flask.cached(timeout=-1, + refreshif=lambda: util.flask.cache_check_headers() or "_refresh" in request.values, key=lambda: "view/%s/%s" % (request.path, g.locale), unless_response=util.flask.cache_check_response_headers) def index(): @@ -401,7 +402,9 @@ def robotsTxt(): @app.route("/i18n//.js") -@util.flask.cached(refreshif=lambda: util.flask.cache_check_headers() or "_refresh" in request.values, key=lambda: "view/%s/%s" % (request.path, g.locale)) +@util.flask.cached(timeout=-1, + refreshif=lambda: util.flask.cache_check_headers() or "_refresh" in request.values, + key=lambda: "view/%s/%s" % (request.path, g.locale)) def localeJs(locale, domain): messages = dict() plural_expr = None From 9ed79adb8dd44bcf59761123f5a7263166dacede Mon Sep 17 00:00:00 2001 From: Salandora Date: Tue, 17 Nov 2015 19:32:34 +0100 Subject: [PATCH 29/66] Fix SD Support deactivated bug and added "unknown command" to the whitelist of errors --- src/octoprint/util/comm.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src/octoprint/util/comm.py b/src/octoprint/util/comm.py index a0308fda4..9023a0067 100644 --- a/src/octoprint/util/comm.py +++ b/src/octoprint/util/comm.py @@ -262,6 +262,7 @@ def __init__(self, port = None, baudrate=None, callbackObject=None, printerProfi self._serial_factory_hooks = self._pluginManager.get_hooks("octoprint.comm.transport.serial.factory") # SD status data + self._sdEnabled = settings().getBoolean(["feature", "sdSupport"]) self._sdAvailable = False self._sdFileList = False self._sdFiles = [] @@ -719,21 +720,28 @@ def getSdFiles(self): return self._sdFiles def startSdFileTransfer(self, filename): - if not self.isOperational() or self.isBusy(): + if not self._sdEnabled: return + if not self.isOperational() or self.isBusy(): + return self._changeState(self.STATE_TRANSFERING_FILE) self.sendCommand("M28 %s" % filename.lower()) def endSdFileTransfer(self, filename): - if not self.isOperational() or self.isBusy(): + if not self._sdEnabled: return + if not self.isOperational() or self.isBusy(): + return self.sendCommand("M29 %s" % filename.lower()) self._changeState(self.STATE_OPERATIONAL) self.refreshSdFiles() def deleteSdFile(self, filename): + if not self._sdEnabled: + return + if not self.isOperational() or (self.isBusy() and isinstance(self._currentFile, PrintingSdFileInformation) and self._currentFile.getFilename() == filename): @@ -744,13 +752,21 @@ def deleteSdFile(self, filename): self.refreshSdFiles() def refreshSdFiles(self): + if not self._sdEnabled: + return + if not self.isOperational() or self.isBusy(): return + self.sendCommand("M20") def initSdCard(self): + if not self._sdEnabled: + return + if not self.isOperational(): return + self.sendCommand("M21") if settings().getBoolean(["feature", "sdAlwaysAvailable"]): self._sdAvailable = True @@ -758,6 +774,9 @@ def initSdCard(self): self._callback.on_comm_sd_state_change(self._sdAvailable) def releaseSdCard(self): + if not self._sdEnabled: + return + if not self.isOperational() or (self.isBusy() and self.isSdFileSelected()): # do not release the sd card if we are currently printing from it return @@ -1331,6 +1350,9 @@ def _handleErrors(self, line): or "error writing to file" in line.lower(): #Also skip errors with the SD card pass + elif 'unknown command' in line.lower(): + #Ignore unkown command errors, it could be a typo or some missing feature + pass elif not self.isError(): self._errorValue = line[6:] if line.startswith("Error:") else line[2:] self._changeState(self.STATE_ERROR) From 7820c17d087dd02573bca5f3dde32229ec9d107c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gina=20H=C3=A4u=C3=9Fge?= Date: Thu, 19 Nov 2015 11:42:12 +0100 Subject: [PATCH 30/66] [Cura] Make plugin a bit more verbose regarding usage & setup --- src/octoprint/plugins/cura/__init__.py | 13 ++++++------ .../cura/templates/cura_settings.jinja2 | 20 ++++++++++++++++++- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/src/octoprint/plugins/cura/__init__.py b/src/octoprint/plugins/cura/__init__.py index fd326a109..095a274b5 100644 --- a/src/octoprint/plugins/cura/__init__.py +++ b/src/octoprint/plugins/cura/__init__.py @@ -37,11 +37,10 @@ def __init__(self): ##~~ TemplatePlugin API - def get_template_configs(self): - from flask.ext.babel import gettext - return [ - dict(type="settings", name=gettext("CuraEngine")) - ] + def get_template_vars(self): + return dict( + homepage=__plugin_url__ + ) ##~~ StartupPlugin API @@ -413,9 +412,9 @@ def _sanitize_name(name): sanitized_name = sanitized_name.replace(" ", "_") return sanitized_name.lower() -__plugin_name__ = "CuraEngine" +__plugin_name__ = "CuraEngine (<= 15.04)" __plugin_author__ = "Gina Häußge" __plugin_url__ = "https://github.com/foosel/OctoPrint/wiki/Plugin:-Cura" -__plugin_description__ = "Adds support for slicing via CuraEngine from within OctoPrint" +__plugin_description__ = "Adds support for slicing via CuraEngine versions up to and including version 15.04 from within OctoPrint" __plugin_license__ = "AGPLv3" __plugin_implementation__ = CuraPlugin() diff --git a/src/octoprint/plugins/cura/templates/cura_settings.jinja2 b/src/octoprint/plugins/cura/templates/cura_settings.jinja2 index 6779f95c2..92cfe0ae0 100644 --- a/src/octoprint/plugins/cura/templates/cura_settings.jinja2 +++ b/src/octoprint/plugins/cura/templates/cura_settings.jinja2 @@ -1,5 +1,12 @@

{{ _('General') }}

+

{% trans %} + Specify the path to the CuraEngine binary. Note that only + versions up to and including 15.04 are supported. + CuraEngine version 15.06 or newer is not + compatible with this plugin. +{% endtrans %}

+
@@ -10,7 +17,7 @@
@@ -53,6 +60,10 @@ +
+ {% trans %}For more information on configuration and usage please see the Plugin's homepage.{% endtrans %} +
+ + + {% trans %} + You can import your existing profile .ini files from Cura (version up to and + including 15.04) here. Please be aware that neither the .json profile format + from Cura versions starting with 15.06 is supported, nor are the custom Cura profile formats + that third party tools like e.g. Repetier create. + {% endtrans %}