SYS-518 break apart monolithic openapi.yaml

.gitignore

@@ -12,6 +12,7 @@ dist
 env-config.js
 htmlcov
 node_modules
+openapi.yaml
 python_env
 results.xml
 stubs

Makefile

@@ -42,10 +42,21 @@ $(VDIR)/bin/python3:
 	@echo "Creating virtual environment"
 	python3 -m venv --system-site-packages $(VENV)
 
+media/openapi.yaml: $(wildcard media/openapi/*.yaml)
+	@echo "Generating openapi.yaml"
+	. $(VDIR)/bin/activate && dref media/openapi/api.yaml $@
+
 flake8: test_requirements
 	@echo "Running flake8 code analysis"
 	. $(VDIR)/bin/activate && flake8 media tests
 
+# for the create_image rule, do this instead for openapi.yaml
+openapi_deploy:
+	pip install dollar-ref
+	ls -l $(find . -name dref)
+	/build/.local/bin/dref media/openapi/api.yaml media/openapi.yaml
+	chmod 644 media/openapi.yaml
+
 $(VDIR)/lib/python3.6/site-packages/pytest.py: python_env
 	@echo "Installing test requirements"
 	(. $(VDIR)/bin/activate && pip3 freeze && \
@@ -58,7 +69,7 @@ py_requirements: $(VDIR)/lib/python3.6/site-packages/flask/app.py
 test_requirements: $(VDIR)/lib/python3.6/site-packages/pytest.py
 
 test: test_requirements py_requirements media/.proto.sqlite \
-	    media/i18n/en/LC_MESSAGES/messages.mo
+	    media/i18n/en/LC_MESSAGES/messages.mo media/openapi.yaml
 	@echo "Running pytest unit tests"
 	cd media && \
 	(. $(VDIR)/bin/activate && \
@@ -84,12 +95,12 @@ media/i18n/en/LC_MESSAGES/messages.mo:
 clean:
 	rm -rf build dist *.egg-info .cache .pytest_cache */__pycache__ \
 	 */.coverage */.proto.sqlite */coverage.xml */htmlcov */results.xml \
-	 docs/_build docs/content/stubs
+	 docs/_build docs/content/stubs media/openapi.yaml
 	find . -name '*.pyc' -or -name '*~' -exec rm -rf {} \;
 wipe_clean: clean
 	rm -rf python_env
 
-create_image: qemu i18n_deploy
+create_image: qemu i18n_deploy openapi_deploy
 	@echo docker build -t $(REGISTRY)/$(IMGNAME)-$(CI_JOB_NAME):$(TAG)
 	@docker buildx build \
 	 --tag $(REGISTRY)/$(IMGNAME)-$(CI_JOB_NAME):$(TAG) . \
@@ -98,7 +109,7 @@ create_image: qemu i18n_deploy
 	 --build-arg=TAG=$(TAG) \
 	 --build-arg=BUILD_DATE=$(shell date +%Y-%m-%dT%H:%M:%SZ)
 
-promote_images: qemu i18n_deploy
+promote_images: qemu i18n_deploy openapi_deploy
 ifeq ($(CI_COMMIT_TAG),)
 	$(foreach target, $(IMAGES), \
 	  image=$(shell basename $(target)) && \

docs/content/layout.rst

@@ -20,7 +20,11 @@ This repository provides the layout for a typical Dockerized microservice. Here'
     │   │   ├── base.py          # declarative base
     │   │   └── media.py         # open-core media models
     │   ├── models               # models for database schema
-    │   ├── openapi.yaml         # OpenAPI 3.0 specifications
+    │   ├── openapi              # OpenAPI 3.0 specifications
+    │   │   ├── api.yaml         # top-level API spec
+    │   │   ├── album.schema.yaml  # open-core API model schema
+    │   │   ├── album.path.yaml    # open-core API routing paths
+    │   │   └── ...              # add your custom specs here
     │   ├── rbac.yaml            # endpoint permissions
     │   ├── _version.py          # application version
     │   └── controllers          # controller classes

docs/requirements.txt

@@ -1,33 +1,33 @@
 autodoc==0.5.0
-pymdown-extensions==7.1
-Sphinx==3.1.1
-sphinxcontrib-mermaid==0.4.0
-sphinx-markdown-parser==0.2.3
+pymdown-extensions==8.1
+Sphinx==3.4.1
+sphinxcontrib-mermaid==0.5.0
+sphinx-markdown-parser==0.2.4
 sphinx-markdown-tables==0.0.15
-sphinx-rtd-theme==0.5.0
+sphinx-rtd-theme==0.5.1
 
 -r ../requirements.txt
 
 # transitive dependencies
 alabaster==0.7.12
-attrs==19.3.0
+attrs==20.3.0
 Babel==2.8.0
-beautifulsoup4==4.9.1
-certifi==2020.6.20
-cffi==1.14.0
+beautifulsoup4==4.9.3
+certifi==2020.12.5
+cffi==1.14.4
 chardet==3.0.4
 click==7.1.2
 commonmark==0.9.1
 idna==2.10
 imagesize==1.2.0
 itsdangerous==1.1.0
 jsonschema==3.2.0
-Markdown==3.2.2
-packaging==20.4
+Markdown==3.3.3
+packaging==20.8
 pycparser==2.20
 pycryptodomex==3.9.8
-pydash==4.8.0
-Pygments==2.6.1
+pydash==4.9.2
+Pygments==2.7.3
 pyparsing==2.4.7
 requests==2.23.0
 six==1.15.0

media/_version.py

@@ -1,3 +1,3 @@
-__version__ = '0.0.26'
+__version__ = '0.0.27'
 vcs_ref = 'unset'
 build_date = 'unset'

media/config.yaml

@@ -4,3 +4,4 @@ appname: Media
 db_migrate_enable: false
 db_schema_maxtime: 0
 service_name: media
+thumbnail_tiny: 50

media/main.py

@@ -12,7 +12,7 @@
 import os
 
 from apicrud import AccessControl, AccountSettings, ServiceConfig, \
-    ServiceRegistry, SessionManager, database, utils
+    ServiceRegistry, SessionManager, database, initialize
 
 import controllers
 import models
@@ -22,7 +22,7 @@
 config = ServiceConfig(
     babel_translation_directories='i18n;%s' % os.path.join(path, 'i18n'),
     reset=True, file=os.path.join(path, 'config.yaml'), models=models).config
-utils.initialize_app(application)
+initialize.app(application)
 babel = Babel(application.app)
 
 

media/media_worker.py

@@ -38,9 +38,9 @@ def incoming(uid, file_id):
     logging.info("action=incoming uid=%s name=%s file_id=%s " % (
         uid, media.meta["name"], file_id))
     if media.meta["ctype"] in constants.MIME_IMAGE_TYPES:
-        media.photo(uid, media.meta, db_session)
+        media.photo(uid, media.meta)
     elif media.meta["ctype"] in constants.MIME_VIDEO_TYPES:
-        media.video(uid, media.meta, db_session)
+        media.video(uid, media.meta)
     else:
         # TODO heic will require another library
         #  https://stackoverflow.com/questions/54395735/how-to-work-with-heic-image-file-types-in-python

media/models/api.py

@@ -12,8 +12,8 @@
      TEXT, TIMESTAMP, Unicode, UniqueConstraint
 from sqlalchemy import func
 from sqlalchemy.orm import relationship, backref
-from sqlalchemy_utils import EncryptedType
-from sqlalchemy_utils.types.encrypted.encrypted_type import AesEngine
+from sqlalchemy_utils.types.encrypted.encrypted_type import AesEngine, \
+    StringEncryptedType
 
 import constants
 from .base import aes_secret, AsDictMixin, Base
@@ -31,12 +31,12 @@ class Account(AsDictMixin, Base):
     name = Column(String(32), nullable=False, unique=True)
     uid = Column(ForeignKey(u'people.id', ondelete='CASCADE'), nullable=False,
                  unique=True)
-    password = Column(EncryptedType(Unicode, aes_secret, AesEngine, 'pkcs5'),
-                      nullable=False)
+    password = Column(StringEncryptedType(Unicode, aes_secret, AesEngine,
+                                          'pkcs5', length=128), nullable=False)
     password_must_change = Column(BOOLEAN, nullable=False,
                                   server_default="False")
-    totp_secret = Column(EncryptedType(Unicode, aes_secret, AesEngine,
-                                       'pkcs5'))
+    totp_secret = Column(StringEncryptedType(Unicode, aes_secret, AesEngine,
+                                             'pkcs5', length=48))
     is_admin = Column(BOOLEAN, nullable=False, server_default="False")
     settings_id = Column(ForeignKey(u'settings.id'), nullable=False)
     last_login = Column(TIMESTAMP)
@@ -51,6 +51,51 @@ class Account(AsDictMixin, Base):
     settings = relationship('Settings')
 
 
+class APIkey(AsDictMixin, Base):
+    __tablename__ = 'apikeys'
+    __table_args__ = (
+        UniqueConstraint(u'uid', u'name', name='uniq_apikey_owner'),
+    )
+    __rest_exclude__ = ('hashvalue',)
+    __rest_related__ = ('scopes',)
+
+    id = Column(String(16), primary_key=True, unique=True)
+    name = Column(String(32), nullable=False)
+    uid = Column(ForeignKey(u'people.id', ondelete='CASCADE'), nullable=False,
+                 unique=True)
+    prefix = Column(String(8), nullable=False, unique=True)
+    hashvalue = Column(StringEncryptedType(Unicode, aes_secret, AesEngine,
+                                           'pkcs5', length=96), nullable=False)
+    expires = Column(TIMESTAMP)
+    last_used = Column(TIMESTAMP)
+    created = Column(TIMESTAMP, nullable=False, server_default=func.now())
+    modified = Column(TIMESTAMP)
+    status = Column(Enum(u'active', u'disabled'), nullable=False)
+
+    scopes = relationship('Scope', secondary='apikeyscopes',
+                          backref=backref('scope'), order_by='Scope.name')
+    owner = relationship('Person', foreign_keys=[uid], backref=backref(
+        'apikey_uid', cascade='all, delete-orphan'))
+
+
+class APIkeyScope(Base):
+    __tablename__ = 'apikeyscopes'
+    __table_args__ = (
+        UniqueConstraint(u'apikey_id', u'scope_id', name='uniq_apikey_scope'),
+    )
+
+    apikey_id = Column(ForeignKey(u'apikeys.id', ondelete='CASCADE'),
+                       primary_key=True, nullable=False)
+    scope_id = Column(ForeignKey(u'scopes.id', ondelete='CASCADE'),
+                      primary_key=True, nullable=False, index=True)
+    created = Column(TIMESTAMP, nullable=False, server_default=func.now())
+
+    apikey = relationship('APIkey', foreign_keys=[apikey_id], backref=backref(
+        'apikeyscopes', cascade='all, delete-orphan'))
+    scope = relationship('Scope', backref=backref(
+        'apikeyscopes', cascade='all, delete-orphan'))
+
+
 class Category(AsDictMixin, Base):
     __tablename__ = 'categories'
     __table_args__ = (
@@ -109,8 +154,10 @@ class Credential(AsDictMixin, Base):
     type = Column(String(16))
     url = Column(String(length=64))
     key = Column(String(length=128))
-    secret = Column(EncryptedType(Unicode, aes_secret, AesEngine, 'pkcs5'))
-    otherdata = Column(EncryptedType(Unicode, aes_secret, AesEngine, 'pkcs5'))
+    secret = Column(StringEncryptedType(Unicode, aes_secret, AesEngine,
+                                        'pkcs5', length=128))
+    otherdata = Column(StringEncryptedType(Unicode, aes_secret, AesEngine,
+                                           'pkcs5', length=1024))
     expires = Column(TIMESTAMP)
     settings_id = Column(ForeignKey(u'settings.id'), nullable=False)
     uid = Column(ForeignKey(u'people.id', ondelete='CASCADE'), nullable=False)
@@ -328,6 +375,21 @@ class Profile(AsDictMixin, Base):
     tz = relationship('Tz')
 
 
+class Scope(AsDictMixin, Base):
+    __tablename__ = 'scopes'
+    __table_args__ = (
+        UniqueConstraint(u'name', u'settings_id', name='uniq_scope_name'),
+    )
+
+    id = Column(String(16), primary_key=True, unique=True)
+    name = Column(String(32), nullable=False)
+    settings_id = Column(ForeignKey(u'settings.id'), nullable=False)
+    created = Column(TIMESTAMP, nullable=False, server_default=func.now())
+    modified = Column(TIMESTAMP)
+    status = Column(Enum('active', u'disabled'), nullable=False,
+                    server_default=u'active')
+
+
 class Settings(AsDictMixin, Base):
     __tablename__ = 'settings'
 

media/models/media.py

@@ -15,8 +15,8 @@
      UniqueConstraint
 from sqlalchemy import func
 from sqlalchemy.orm import relationship, backref
-from sqlalchemy_utils import EncryptedType
-from sqlalchemy_utils.types.encrypted.encrypted_type import AesEngine
+from sqlalchemy_utils.types.encrypted.encrypted_type import AesEngine, \
+    StringEncryptedType
 
 import constants
 from .base import aes_secret, AsDictMixin, Base
@@ -34,7 +34,8 @@ class Album(AsDictMixin, Base):
     sizes = Column(String(32), nullable=False,
                    server_default=constants.DEFAULT_THUMBNAIL_SIZES)
     encryption = Column(Enum(u'aes'))
-    password = Column(EncryptedType(Unicode, aes_secret, AesEngine, 'pkcs5'))
+    password = Column(StringEncryptedType(Unicode, aes_secret, AesEngine,
+                                          'pkcs5', length=64))
     uid = Column(ForeignKey(u'people.id', ondelete='CASCADE'), nullable=False)
     list_id = Column(ForeignKey(u'lists.id'))
     event_id = Column(ForeignKey(u'events.id'))