Improve Project's Layout (#66)

Move parsers into own python package. Using a dedicated folder for parsers makes the project structure more clear and simplifies to find stuff. In addition it makes it easier to identifier individual parsers and tools and infrastructure used by all parsers.

Extend setup.py to used setuptools to find packages. Top modules are now listed manually.

Travis build scripts and the urls that are tested are now grouped under `build_scripts`. 

The `parse` method is move `parse.py`. `config.py` simply defines which parsers exists.

Many thanks to @y0hy0h for the time and effort of these changes.

.travis.yml

@@ -5,18 +5,18 @@ dist: trusty
 before_install:
   - deactivate
 install:
-  - sudo scripts/travis-install-deps.sh
+  - sudo build_scripts/travis-install-deps.sh
 before_script:
-  - scripts/travis-prepare-packaging.sh
+  - build_scripts/travis-prepare-packaging.sh
 script:
   - dpkg-buildpackage -uc -us
   - dpkg --info ../*.deb
   - dpkg --contents ../*.deb
-  - sudo scripts/travis-install-and-setup-deb.sh
-  - wget --output-document=/dev/null --input-file=test-urls.txt
-  - wget --output-document=/dev/null --input-file=maybe-urls.txt || true
+  - sudo build_scripts/travis-install-and-setup-deb.sh
+  - wget --output-document=/dev/null --input-file=build_scripts/test-urls.txt
+  - wget --output-document=/dev/null --input-file=build_scripts/maybe-urls.txt || true
 after_success:
-  - 'test $TRAVIS_PULL_REQUEST == "false" && test $TRAVIS_BRANCH == "master" && scripts/travis-publish-package.sh'
+  - 'test $TRAVIS_PULL_REQUEST == "false" && test $TRAVIS_BRANCH == "master" && build_scripts/travis-publish-package.sh'
 after_script:
   - sudo cat /var/log/uwsgi/app/openmensa-parsers.log
 env:

build_scripts/travis-install-deps.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -ex
+apt-get update -qq
+apt-get install -qq python3 python3-setuptools python3-bs4 python3-lxml uwsgi uwsgi-plugin-python3 devscripts debhelper

config.py

@@ -1,23 +1,8 @@
 import importlib
 
-from utils import ParserNotFound, CanteenPrefixer, ParserRenamer
+from utils import CanteenPrefixer, ParserRenamer
 
-
-parsers = {}
-
-
-def register_parser(parser):
-    parsers[parser.name] = parser
-
-
-def parse(request, parser_name, *args):
-    if parser_name in parsers:
-        return parsers[parser_name].parse(request, *args)
-    else:
-        raise ParserNotFound(parser_name)
-
-
-for module in [
+cities = [
     'aachen',
     'chemnitz_zwickau',
     'darmstadt',
@@ -37,14 +22,27 @@ def parse(request, parser_name, *args):
     'siegen',
     'wuerzburg',
     'rostock',
-]:
-    register_parser(importlib.import_module(module).parser)
+]
+
+
+def register_all_parsers(module_list):
+    registered_parsers = {}
+
+    def register_parser(parser):
+        registered_parsers[parser.name] = parser
+
+    for module in module_list:
+        register_parser(importlib.import_module('parsers.' + module).parser)
+
+    register_parser(CanteenPrefixer('braunschweig', 'ostniedersachsen'))
+    register_parser(ParserRenamer('clausthal', 'ostniedersachsen'))
+    register_parser(CanteenPrefixer('hildesheim', 'ostniedersachsen'))
+    register_parser(CanteenPrefixer('suderburg', 'ostniedersachsen'))
+    register_parser(CanteenPrefixer('wolfenbuettel', 'ostniedersachsen'))
+    register_parser(CanteenPrefixer('holzminden', 'ostniedersachsen'))
+    register_parser(CanteenPrefixer('lueneburg', 'ostniedersachsen'))
+
+    return registered_parsers
 
 
-register_parser(CanteenPrefixer('braunschweig', 'ostniedersachsen'))
-register_parser(ParserRenamer('clausthal', 'ostniedersachsen'))
-register_parser(CanteenPrefixer('hildesheim', 'ostniedersachsen'))
-register_parser(CanteenPrefixer('suderburg', 'ostniedersachsen'))
-register_parser(CanteenPrefixer('wolfenbuettel', 'ostniedersachsen'))
-register_parser(CanteenPrefixer('holzminden', 'ostniedersachsen'))
-register_parser(CanteenPrefixer('lueneburg', 'ostniedersachsen'))
+parsers = register_all_parsers(cities)

parse.py

@@ -1,17 +1,25 @@
 #!/usr/bin/env python3
 import sys
 
-from config import parse
-from utils import Request, NotFoundError
+from config import parsers
+from utils import Request, NotFoundError, ParserNotFound
+
+
+def parse(request, parser_name, *args):
+    if parser_name in parsers:
+        return parsers[parser_name].parse(request, *args)
+    else:
+        raise ParserNotFound(parser_name)
 
 
 class SimulatedRequest(Request):
     def __init__(self):
         self.host = 'http://example.org'
 
 
-try:
-    print(parse(SimulatedRequest(), *sys.argv[1:]))
-except NotFoundError as e:
-    print(e)
-    sys.exit(2)
+if __name__ == '__main__':
+    try:
+        print(parse(SimulatedRequest(), *sys.argv[1:]))
+    except NotFoundError as e:
+        print(e)
+        sys.exit(2)

setup.py

@@ -1,15 +1,14 @@
-import os
-import glob
 from distutils.core import setup
 
-modules = list(map(lambda d: d[:-3], glob.glob(os.path.join('*.py'))))
-modules.remove('setup')
+from setuptools import find_packages
 
 setup(name="openmensa-parsers",
       version="1.0",
       description="A collection with parsers for openmensa",
       author="Malte Swart",
       author_email="mswart@devtation.de",
       url="https://github.com/mswart/openmensa-parsers.git",
-      py_modules=modules + ['pyopenmensa/__init__', 'pyopenmensa/feed'],
+      packages=find_packages(),
+      py_modules=['config', 'utils', 'parse', 'wsgihandler', 'pyopenmensa/__init__',
+                  'pyopenmensa/feed'],
       requires=['beautifulsoup4', 'lxml'])

wsgihandler.py

@@ -2,7 +2,7 @@
 import traceback
 import re
 
-from config import parse
+from parse import parse
 import utils
 
 canteen_request = re.compile('/(?P<dirs>([\w-]+/)*[\w-]+)/(?P<file>[\w-]+.(xml|json))')