Merge pull request #660 from tahoe-lafs/3256.tahoe-ported-filename

Fix the "tahoe-ported.json" filename in the depgraph script

.travis.yml

@@ -25,7 +25,7 @@ script:
   - |
     set -eo pipefail
     if [ "${T}" = "py35" ]; then
-      python3 -m compileall -f .
+      python3 -m compileall -f -x tahoe-depgraph.py .
     else
       tox -e ${T}
     fi

misc/python3/depgraph.sh

@@ -3,17 +3,12 @@
 set -x
 set -eo pipefail
 
-if [ "${CIRCLE_BRANCH}" != "master" ]; then
-    echo "Declining to update dependency graph for non-master build."
-    exit 0
-fi
-
 TAHOE="${PWD}"
 git clone -b gh-pages git@github.com:tahoe-lafs/tahoe-depgraph.git
 cd tahoe-depgraph
 
 # Generate the maybe-changed data.
-python tahoe-depgraph.py "${TAHOE}"
+python "${TAHOE}"/misc/python3/tahoe-depgraph.py "${TAHOE}"
 
 if git diff-index --quiet HEAD; then
   echo "Declining to commit without any changes."
@@ -23,12 +18,17 @@ fi
 git config user.name 'Build Automation'
 git config user.email 'tahoe-dev@tahoe-lafs.org'
 
-git add tahoe-deps.json tahoe-ported.json.
+git add tahoe-deps.json tahoe-ported.json
 git commit -m "\
 Built from ${CIRCLE_REPOSITORY_URL}@${CIRCLE_SHA1}
 
 tahoe-depgraph was $(git rev-parse HEAD)
 "
 
+if [ "${CIRCLE_BRANCH}" != "master" ]; then
+    echo "Declining to update dependency graph for non-master build."
+    exit 0
+fi
+
 # Publish it on GitHub.
 git push -q origin gh-pages

misc/python3/tahoe-depgraph.py

@@ -0,0 +1,123 @@
+# Copyright 2004, 2009 Toby Dickenson
+# Copyright 2014-2015 Aaron Gallagher
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject
+# to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+import collections
+import functools
+import json
+import os
+import modulefinder
+import sys
+import tempfile
+
+from twisted.python import reflect
+
+
+class mymf(modulefinder.ModuleFinder):
+    def __init__(self, *args, **kwargs):
+        self._depgraph = collections.defaultdict(set)
+        self._types = {}
+        self._last_caller = None
+        modulefinder.ModuleFinder.__init__(self, *args, **kwargs)
+
+    def import_hook(self, name, caller=None, fromlist=None, level=None):
+        old_last_caller = self._last_caller
+        try:
+            self._last_caller = caller
+            return modulefinder.ModuleFinder.import_hook(
+                self, name, caller, fromlist)
+        finally:
+            self._last_caller = old_last_caller
+
+    def import_module(self, partnam, fqname, parent):
+        if partnam.endswith('_py3'):
+            return None
+        r = modulefinder.ModuleFinder.import_module(
+            self, partnam, fqname, parent)
+        last_caller = self._last_caller
+        if r is not None and 'allmydata' in r.__name__:
+            if last_caller is None or last_caller.__name__ == '__main__':
+                self._depgraph[fqname]
+            else:
+                self._depgraph[last_caller.__name__].add(fqname)
+        return r
+
+    def load_module(self, fqname, fp, pathname, (suffix, mode, type)):
+        r = modulefinder.ModuleFinder.load_module(
+            self, fqname, fp, pathname, (suffix, mode, type))
+        if r is not None:
+            self._types[r.__name__] = type
+        return r
+
+    def as_json(self):
+        return {
+            'depgraph': {
+                name: dict.fromkeys(deps, 1)
+                for name, deps in self._depgraph.iteritems()},
+            'types': self._types,
+        }
+
+
+json_dump = functools.partial(
+    json.dump, indent=4, separators=(',', ': '), sort_keys=True)
+
+
+def main(target):
+    mf = mymf(sys.path[:], 0, [])
+
+    moduleNames = []
+    for path, dirnames, filenames in os.walk(os.path.join(target, 'src', 'allmydata')):
+        if 'test' in dirnames:
+            dirnames.remove('test')
+        for filename in filenames:
+            if not filename.endswith('.py'):
+                continue
+            if filename in ('setup.py',):
+                continue
+            if '-' in filename:
+                # a script like update-documentation.py
+                continue
+            if filename != '__init__.py':
+                filepath = os.path.join(path, filename)
+            else:
+                filepath = path
+            moduleNames.append(reflect.filenameToModuleName(filepath))
+
+    with tempfile.NamedTemporaryFile() as tmpfile:
+        for moduleName in moduleNames:
+            tmpfile.write('import %s\n' % moduleName)
+        tmpfile.flush()
+        mf.run_script(tmpfile.name)
+
+    with open('tahoe-deps.json', 'wb') as outfile:
+        json_dump(mf.as_json(), outfile)
+        outfile.write('\n')
+
+    ported_modules_path = os.path.join(target, "misc", "python3", "ported-modules.txt")
+    with open(ported_modules_path) as ported_modules:
+        port_status = dict.fromkeys((line.strip() for line in ported_modules), "ported")
+    with open('tahoe-ported.json', 'wb') as outfile:
+        json_dump(port_status, outfile)
+        outfile.write('\n')
+
+
+if __name__ == '__main__':
+    main(*sys.argv[1:])