Make Kibble a package (#67)

This commit refactors the code base to create an installable
kibble application.

Co-authored-by: Michał Słowikowski <michalslowikowski00@gmail.com>

.gitignore

@@ -1,5 +1,6 @@
 # Apache Kibble files
 api/yaml/kibble.yaml
+kibble/api/yaml/kibble.yaml
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

CONTRIBUTING.md

@@ -15,6 +15,11 @@ We also have:
 
 ## Development installation
 
+You should be able to install Apache Kibble by simply doing:
+```
+pip install -e ."[devel]"
+```
+
 The easiest option to spin up a development environment is to use our development docker-compose.
 The development image has mounted all Kibble sources so all your local code changes will be automatically
 reflected  in the running app.

Dockerfile.dev

@@ -15,17 +15,15 @@
 # specific language governing permissions and limitations
 # under the License.
 
-FROM python:3.6
+FROM python:3.8
 
 USER root
 RUN apt-get update
 RUN apt-get install -y gcc unzip
 
-COPY ./api /kibble/api/
-COPY ./setup /kibble/setup/
-COPY ./ui /kibble/ui/
-
-RUN pip install --upgrade pip
-RUN pip install -r /kibble/setup/requirements.txt
+COPY . /kibble/
 
 WORKDIR /kibble
+
+RUN pip install --upgrade pip
+RUN pip install -e .

docker-compose-dev.yaml

@@ -7,24 +7,22 @@ services:
     build:
       context: .
       dockerfile: Dockerfile.dev
-    command: bash -c "python setup/setup.py -e elasticsearch -a -k"
+    command: bash -c "python kibble/setup/setup.py -e elasticsearch -a -k"
     volumes:
-      - ./setup/:/kibble/setup/
+      - .:/kibble/
     depends_on:
       - elasticsearch
 
   # Apache Kibble API server
   kibble:
     image: *img
-    command: bash -c "gunicorn --reload -w 1 -b 0.0.0.0:8001 api.handler:application"
+    command: bash -c "gunicorn --reload -w 1 -b 0.0.0.0:8001 kibble.api.handler:application"
     expose:
       - 8001
     ports:
       - 8001:8001
     volumes:
-      - ./api/:/kibble/api/
-      - ./setup/:/kibble/setup/
-      - ./ui/:/kibble/ui/
+      - .:/kibble/
     depends_on:
       - elasticsearch
 

kibble/__main__.py

@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+def main():
+    print("Hello to kibble!")
+
+
+if __name__ == '__main__':
+    main()

kibble/api/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

api/handler.py → kibble/api/handler.py

@@ -30,30 +30,31 @@
 import yaml
 import json
 
-from api.plugins import openapi
-from api.plugins.database import KibbleDatabase
-from api.plugins.session import KibbleSession
+from kibble.api.plugins import openapi
+from kibble.api.plugins.database import KibbleDatabase
+from kibble.api.plugins.session import KibbleSession
 
 
 # Compile valid API URLs from the pages library
 # Allow backwards compatibility by also accepting .lua URLs
+from kibble.settings import KIBBLE_YAML, YAML_DIRECTORY
+
 urls = []
 if __name__ != '__main__':
-    from api.pages import handlers
+    from kibble.api.pages import handlers
     for page, handler in handlers.items():
         urls.append((r"^(/api/%s)(/.+)?$" % page, handler.run))
 
 
 # Load Kibble master configuration
-config_yaml = os.path.join(os.path.dirname(os.path.realpath(__file__)), "yaml", "kibble.yaml")
-with open(config_yaml, "r") as f:
+with open(KIBBLE_YAML, "r") as f:
     config = yaml.load(f)
 
 # Instantiate database connections
 DB = None
 
 # Load Open API specifications
-openapi_yaml = os.path.join(os.path.dirname(os.path.realpath(__file__)), "yaml", "openapi.yaml")
+openapi_yaml = os.path.join(YAML_DIRECTORY, "openapi.yaml")
 KibbleOpenAPI = openapi.OpenAPI(openapi_yaml)
 
 

api/pages/__init__.py → kibble/api/pages/__init__.py

@@ -42,5 +42,5 @@ def loadPage(path):
                 p = filepath.replace(rootpath, "")[1:].replace('/', '.')[:-3]
                 xp = p.replace('.', '/')
                 print("Loading endpoint pages.%s as %s" % (p, xp))
-                handlers[xp] = importlib.import_module("api.pages.%s" % p)
+                handlers[xp] = importlib.import_module(f"kibble.api.pages.{p}")
 loadPage(rootpath)

api/pages/org/sourcetypes.py → kibble/api/pages/org/sourcetypes.py

@@ -68,12 +68,16 @@
 """
 This is the source types handler for Kibble
 """
+import os
 
 import yaml
 import json
 
-def run(API, environ, indata, session):
+from kibble.settings import YAML_DIRECTORY
+
 
-    types = yaml.load(open("yaml/sourcetypes.yaml"))
+def run(API, environ, indata, session):
+    with open(os.path.join(YAML_DIRECTORY, "sourcetypes.yaml")) as f:
+        types = yaml.load(f)
 
     yield json.dumps(types)

api/pages/sources.py → kibble/api/pages/sources.py

@@ -128,11 +128,15 @@
 """
 
 import json
+import os
 import re
 import time
 import hashlib
 import yaml
 
+from kibble.settings import YAML_DIRECTORY
+
+
 def canModifySource(session):
     """ Determine if the user can edit sources in this org """
 
@@ -224,7 +228,8 @@ def run(API, environ, indata, session):
         if canModifySource(session):
             new = 0
             old = 0
-            stypes = yaml.load(open("yaml/sourcetypes.yaml"))
+            with open(os.path.join(YAML_DIRECTORY, "sourcetypes.yaml")) as f:
+                stypes = yaml.load(f)
             for source in indata.get('sources', []):
                 sourceURL = source['sourceURL']
                 sourceType = source['type']

api/pages/widgets.py → kibble/api/pages/widgets.py

@@ -46,16 +46,21 @@
 """
 This is the widget design handler for Kibble
 """
+import os
 
 import yaml
 import json
 
+from kibble.settings import YAML_DIRECTORY
+
+
 def run(API, environ, indata, session):
 
     if not session.user:
         raise API.exception(403, "You must be logged in to use this API endpoint! %s")
 
-    widgets = yaml.load(open("yaml/widgets.yaml"))
+    with open(os.path.join(YAML_DIRECTORY, "widgets.yaml")) as f:
+        widgets = yaml.load(f)
 
     page = indata['pageid']
     if not page or page == '0':

api/yaml/openapi/combine.py → kibble/api/yaml/openapi/combine.py

@@ -20,6 +20,8 @@
 import sys
 import re
 
+from kibble.settings import YAML_DIRECTORY
+
 baseyaml = """
 # THIS IS PULLED FROM SCRIPTS AND AUTOGENERATED!
 # Please use openapi/combine.py to regenerate!
@@ -106,7 +108,8 @@ def construct():
             if fname.endswith(".py"):
                 fpath = "%s/%s" % (apidir, fname)
                 print("Scanning %s" % fpath)
-                contents = open(fpath, "r").read()
+                with open(fpath, "r") as f:
+                    contents = f.read()
                 m = re.search(r"OPENAPI-URI: (\S+)\n##+\n([\s\S]+?)##+", contents)
                 if m:
                     apath = m.group(1)
@@ -128,7 +131,7 @@ def construct():
                     print("Scanning %s" % fpath)
                     defs = yaml.load(open(fpath))
                     yml['components'][d][fname.replace(".yaml", "")] = defs
-    ypath = os.path.abspath("%s/../openapi.yaml" % bpath)
+    ypath = os.path.join(YAML_DIRECTORY, "openapi.yaml")
     with open(ypath, "w") as f:
         f.write(baseyaml)
         f.write(yaml.dump(yml, default_flow_style=False))

kibble/settings.py

@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+
+YAML_DIRECTORY = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)),
+    "api",
+    "yaml",
+)
+KIBBLE_YAML = os.path.join(YAML_DIRECTORY, "kibble.yaml")

setup/makeaccount.py → kibble/setup/makeaccount.py

@@ -21,6 +21,9 @@
 import yaml
 import bcrypt
 
+from kibble.settings import YAML_DIRECTORY, KIBBLE_YAML
+
+
 class KibbleDatabase(object):
     def __init__(self, config):
         self.config = config
@@ -49,7 +52,8 @@ def __init__(self, config):
 args = arg_parser.parse_args()
 
 # Load Kibble master configuration
-config = yaml.load(open("../api/yaml/kibble.yaml"))
+with open(KIBBLE_YAML) as f:
+    config = yaml.load(f)
 
 DB = KibbleDatabase(config)
 

setup/setup.py → kibble/setup/setup.py

@@ -29,6 +29,8 @@
 import json
 from elasticsearch import Elasticsearch
 
+from kibble.settings import KIBBLE_YAML
+
 KIBBLE_VERSION = '0.1.0'  # ABI/API compat demarcation.
 KIBBLE_DB_VERSION = 2  # Second database revision
 
@@ -219,13 +221,7 @@ def create_es_index(
 
 def get_kibble_yaml() -> str:
     """Resolve path to kibble config yaml"""
-    kibble_yaml = os.path.join(
-        os.path.dirname(os.path.realpath(__file__)),
-        os.pardir,
-        "api",
-        "yaml",
-        "kibble.yaml"
-    )
+    kibble_yaml = KIBBLE_YAML
     if os.path.exists(kibble_yaml):
         print(f"{kibble_yaml} already exists! Writing to {kibble_yaml}.tmp instead")
         kibble_yaml = kibble_yaml + ".tmp"

kibble/version.py

@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+version = "1.0.0dev"

setup.cfg

@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[metadata]
+name = Kibble
+summary = Apache Kibble is a tool to collect, aggregate and visualize data about any software project that uses commonly known tools.
+description-file = README.md
+author = Apache Kibble
+author-email = dev@kibble.apache.org
+license = Apache License, Version 2.0
+license_files =
+   LICENSE
+   NOTICE
+
+[bdist_wheel]
+python-tag=py3
+
+
+[files]
+packages = kibble
+
+[easy_install]