Support nested namespaces (without ~prefix)

klaus/__init__.py

@@ -7,6 +7,7 @@
 import flask
 import httpauth
 import dulwich.web
+from werkzeug.exceptions import NotFound
 from dulwich.errors import NotGitRepository
 from klaus import views, utils
 from klaus.repo import FancyRepo, InvalidRepo
@@ -15,6 +16,59 @@
 KLAUS_VERSION = utils.guess_git_revision() or "1.5.2"
 
 
+class KlausRedirects(flask.Flask):
+    def __init__(self, repos):
+        flask.Flask.__init__(self, __name__)
+
+        for namespaced_name in repos:
+            self.setup_redirects('/' + namespaced_name)
+            if namespaced_name.count('/') == 1:
+                self.setup_redirects('/' + namespaced_name, '/~' + namespaced_name)
+
+    def query_str(self):
+      query = flask.request.query_string.decode()
+      if len(query) > 0:
+          return '?' + query
+
+      return ''
+
+    def setup_redirects(self, route, pattern=None):
+        if not pattern:
+            pattern = route
+
+        def redirect_root():
+            return flask.redirect(route + '/-/' + self.query_str(), 301)
+
+        def redirect_rest(path):
+            if path.startswith('-/'):
+                raise NotFound()
+            return flask.redirect(route + '/-/' + path + self.query_str(), 301)
+
+        def redirect_git():
+            return flask.redirect(route + '.git/info/refs' + self.query_str(), 301)
+
+        self.add_url_rule(
+            pattern + '/',
+            endpoint=pattern + '_root',
+            view_func=redirect_root,
+        )
+        self.add_url_rule(
+            pattern + '.git',
+            endpoint=pattern + '_git2root',
+            view_func=redirect_root,
+        )
+        self.add_url_rule(
+            pattern + '/<path:path>',
+            endpoint=pattern + '_rest',
+            view_func=redirect_rest,
+        )
+        self.add_url_rule(
+            pattern + '/info/refs',
+            endpoint=pattern + '_git',
+            view_func=redirect_git,
+        )
+
+
 class Klaus(flask.Flask):
     jinja_options = {
         "extensions": [] if jinja2_autoescape_builtin else ["jinja2.ext.autoescape"],
@@ -25,6 +79,7 @@ def __init__(self, repo_paths, site_name, use_smarthttp, ctags_policy="none"):
         """(See `make_app` for parameter descriptions.)"""
         self.site_name = site_name
         self.use_smarthttp = use_smarthttp
+        self.smarthttp = None # dulwich wsgi app
         self.ctags_policy = ctags_policy
 
         valid_repos, invalid_repos = self.load_repos(repo_paths)
@@ -55,33 +110,48 @@ def create_jinja_environment(self):
         return env
 
     def setup_routes(self):
+        redirects = {}
+
         # fmt: off
         for endpoint, rule in [
             ('repo_list',   '/'),
             ('robots_txt',  '/robots.txt/'),
-            ('blob',        '/<repo>/blob/'),
-            ('blob',        '/<repo>/blob/<rev>/<path:path>'),
-            ('blame',       '/<repo>/blame/'),
-            ('blame',       '/<repo>/blame/<rev>/<path:path>'),
-            ('raw',         '/<repo>/raw/<path:path>/'),
-            ('raw',         '/<repo>/raw/<rev>/<path:path>'),
-            ('submodule',   '/<repo>/submodule/<rev>/'),
-            ('submodule',   '/<repo>/submodule/<rev>/<path:path>'),
-            ('commit',      '/<repo>/commit/<path:rev>/'),
-            ('patch',       '/<repo>/commit/<path:rev>.diff'),
-            ('patch',       '/<repo>/commit/<path:rev>.patch'),
-            ('index',       '/<repo>/'),
-            ('index',       '/<repo>/<path:rev>'),
-            ('history',     '/<repo>/tree/<rev>/'),
-            ('history',     '/<repo>/tree/<rev>/<path:path>'),
-            ('download',    '/<repo>/tarball/<path:rev>/'),
+            ('blob',        '/<repo>/-/blob/'),
+            ('blob',        '/<repo>/-/blob/<rev>/<path:path>'),
+            ('blame',       '/<repo>/-/blame/'),
+            ('blame',       '/<repo>/-/blame/<rev>/<path:path>'),
+            ('raw',         '/<repo>/-/raw/<path:path>/'),
+            ('raw',         '/<repo>/-/raw/<rev>/<path:path>'),
+            ('submodule',   '/<repo>/-/submodule/<rev>/'),
+            ('submodule',   '/<repo>/-/submodule/<rev>/<path:path>'),
+            ('commit',      '/<repo>/-/commit/<path:rev>/'),
+            ('patch',       '/<repo>/-/commit/<path:rev>.diff'),
+            ('patch',       '/<repo>/-/commit/<path:rev>.patch'),
+            ('index',       '/<repo>/-/'),
+            ('index',       '/<repo>/-/<path:rev>'),
+            ('history',     '/<repo>/-/tree/<rev>/'),
+            ('history',     '/<repo>/-/tree/<rev>/<path:path>'),
+            ('download',    '/<repo>/-/tarball/<path:rev>/'),
+            ('smarthttp',   '/<repo>.git'),
         ]:
             self.add_url_rule(rule, view_func=getattr(views, endpoint))
             if "<repo>" in rule:
                 self.add_url_rule(
-                    "/~<namespace>" + rule, view_func=getattr(views, endpoint)
+                    rule.replace('<repo>', '<path:namespace>/<repo>'),
+                    view_func=getattr(views, endpoint)
                 )
         # fmt: on
+        if self.use_smarthttp:
+            self.add_url_rule(
+                '/<repo>.git/<path:path>',
+                view_func=views.smarthttp,
+                methods=['GET', 'POST'],
+            )
+            self.add_url_rule(
+                '/<path:namespace>/<repo>.git/<path:path>',
+                view_func=views.smarthttp,
+                methods=['GET', 'POST'],
+            )
 
     def should_use_ctags(self, git_repo, git_commit):
         if self.ctags_policy == "none":
@@ -167,23 +237,20 @@ def make_app(
         use_smarthttp,
         ctags_policy,
     )
+    app.wsgi_app = utils.ChainedApps(
+        app,
+        KlausRedirects(app.valid_repos),
+    )
     app.wsgi_app = utils.ProxyFix(app.wsgi_app)
 
     if use_smarthttp:
         # `path -> Repo` mapping for Dulwich's web support
-        dulwich_backend = dulwich.server.DictBackend(
-            {
-                "/" + namespaced_name: repo
-                for namespaced_name, repo in app.valid_repos.items()
-            }
-        )
-        # Dulwich takes care of all Git related requests/URLs
-        # and passes through everything else to klaus
-        dulwich_wrapped_app = dulwich.web.make_wsgi_chain(
-            backend=dulwich_backend,
-            fallback_app=app.wsgi_app,
-        )
-        dulwich_wrapped_app = utils.ProxyFix(dulwich_wrapped_app)
+        dulwich_repos = {}
+        for namespaced_name, repo in app.valid_repos.items():
+            dulwich_repos["/" + namespaced_name + '.git'] = repo
+
+        dulwich_backend = dulwich.server.DictBackend(dulwich_repos)
+        dulwich_app = dulwich.web.make_wsgi_chain(backend=dulwich_backend)
 
         # `receive-pack` is requested by the "client" on a push
         # (the "server" is asked to *receive* packs), i.e. we need to secure
@@ -200,31 +267,31 @@ def make_app(
         # failed for /info/refs, but since it's used to upload stuff to the server
         # we must secure it anyway for security reasons.
         PATTERN = (
-            r"^/(~[^/]+/)?[^/]+/(info/refs\?service=git-receive-pack|git-receive-pack)$"
+            r"^/.*\.git/(info/refs\?service=git-receive-pack|git-receive-pack)$"
         )
         if unauthenticated_push:
             # DANGER ZONE: Don't require authentication for push'ing
-            app.wsgi_app = dulwich_wrapped_app
+            app.smarthttp = dulwich_app
         elif htdigest_file and not disable_push:
             # .htdigest file given. Use it to read the push-er credentials from.
             if require_browser_auth:
                 # No need to secure push'ing if we already require HTTP auth
                 # for all of the Web interface.
-                app.wsgi_app = dulwich_wrapped_app
+                app.smarthttp = dulwich_app
             else:
                 # Web interface isn't already secured. Require authentication for push'ing.
-                app.wsgi_app = httpauth.DigestFileHttpAuthMiddleware(
+                app.smarthttp = httpauth.DigestFileHttpAuthMiddleware(
                     htdigest_file,
-                    wsgi_app=dulwich_wrapped_app,
+                    wsgi_app=dulwich_app,
                     routes=[PATTERN],
                 )
         else:
             # No .htdigest file given. Disable push-ing.  Semantically we should
             # use HTTP 403 here but since that results in freaky error messages
             # (see above) we keep asking for authentication (401) instead.
             # Git will print a nice error message after a few tries.
-            app.wsgi_app = httpauth.AlwaysFailingAuthMiddleware(
-                wsgi_app=dulwich_wrapped_app,
+            app.smarthttp = httpauth.AlwaysFailingAuthMiddleware(
+                wsgi_app=dulwich_app,
                 routes=[PATTERN],
             )
 

klaus/contrib/wsgi_autoreload_recursive.py

@@ -0,0 +1,104 @@
+from __future__ import print_function
+import os
+import os.path
+import time
+import threading
+import warnings
+
+from klaus import make_app
+
+# Shared state between poller and application wrapper
+class _:
+    #: the real WSGI app
+    inner_app = None
+    should_reload = True
+
+
+def find_git_repos_recursive(dir):
+    if dir.endswith('.git'):
+        yield dir
+        return
+
+    subdirectories = []
+    for entry in os.scandir(dir):
+        if entry.name == '.git':
+            yield dir
+            return
+
+        if entry.is_dir():
+            subdirectories.append(entry.path)
+
+    for path in subdirectories:
+        yield from find_git_repos_recursive(path)
+
+
+def namespaceify(root, repos):
+    map = {}
+    raw = []
+    map[None] = raw
+    for path in repos:
+        repo = os.path.relpath(path, root)
+        try:
+            [namespace, name] = repo.rsplit('/', 1)
+            map[namespace] = map.get(namespace, [])
+            map[namespace].append(path)
+        except ValueError:
+            raw.append(path)
+    return map
+
+
+def poll_for_changes(interval, dir):
+    """
+    Polls `dir` for changes every `interval` seconds and sets `should_reload`
+    accordingly.
+    """
+    old_contents = list(find_git_repos_recursive(dir))
+    while 1:
+        time.sleep(interval)
+        if _.should_reload:
+            # klaus application has not seen our change yet
+            continue
+        new_contents = find_git_repos_recursive(dir)
+        if new_contents != old_contents:
+            # Directory contents changed => should_reload
+            old_contents = new_contents
+            _.should_reload = True
+
+
+def make_autoreloading_app(repos_root, *args, **kwargs):
+    def app(environ, start_response):
+        if _.should_reload:
+            # Refresh inner application with new repo list
+            print("Reloading repository list...")
+            _.inner_app = make_app(
+                namespaceify(repos_root, find_git_repos_recursive(repos_root)),
+                *args, **kwargs
+            )
+            _.should_reload = False
+        return _.inner_app(environ, start_response)
+
+    # Background thread that polls the directory for changes
+    poller_thread = threading.Thread(target=(lambda: poll_for_changes(10, repos_root)))
+    poller_thread.daemon = True
+    poller_thread.start()
+
+    return app
+
+
+if 'KLAUS_REPOS' in os.environ:
+    warnings.warn("use KLAUS_REPOS_ROOT instead of KLAUS_REPOS for the autoreloader apps", DeprecationWarning)
+
+if 'KLAUS_HTDIGEST_FILE' in os.environ:
+    with open(os.environ['KLAUS_HTDIGEST_FILE']) as file:
+        application = make_autoreloading_app(
+            os.environ.get('KLAUS_REPOS_ROOT') or os.environ['KLAUS_REPOS'],
+            os.environ['KLAUS_SITE_NAME'],
+            os.environ.get('KLAUS_USE_SMARTHTTP'),
+            file,
+        )
+else:
+    application = make_autoreloading_app(
+        os.environ.get('KLAUS_REPOS_ROOT') or os.environ['KLAUS_REPOS'],
+        os.environ['KLAUS_SITE_NAME'],
+        os.environ.get('KLAUS_USE_SMARTHTTP'),
+    )

klaus/repo.py

@@ -1,4 +1,4 @@
-import os
+import os.path
 import io
 import stat
 import subprocess
@@ -43,10 +43,7 @@ def name(self):
 
     @property
     def namespaced_name(self):
-        if self.namespace:
-            return "~{}/{}".format(self.namespace, self.name)
-        else:
-            return self.name
+        return os.path.join(self.namespace or "", self.name)
 
     # TODO: factor out stuff into dulwich
     def get_last_updated_at(self):
@@ -365,7 +362,4 @@ def name(self):
 
     @property
     def namespaced_name(self):
-        if self.namespace:
-            return "~{}/{}".format(self.namespace, self.name)
-        else:
-            return self.name
+        return os.path.join(self.namespace or "", self.name)

klaus/templates/history.inc.html

@@ -35,7 +35,7 @@ <h2>
       @<a href="{{ url_for('index', repo=repo.name, namespace=namespace, rev=rev) }}">{{ rev }}</a>
     </span>
     {% if USE_SMARTHTTP %}
-        <code>git clone {{ url_for('index', repo=repo.name, namespace=namespace, _external=True) }}</code>
+        <code>git clone {{ url_for('smarthttp', repo=repo.name, namespace=namespace, _external=True) }}</code>
     {% endif %}
     {% if repo.cloneurl %}
         <code>git clone {{ repo.cloneurl }}</code>