Userdata Initialization (#2)

.github/workflows/build_and_publish.yml

@@ -0,0 +1,134 @@
+name: Build, publish and deploy docker
+
+on:
+  push:
+    branches: [ 'main' ]
+    tags:
+      - 'v*'
+
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-and-push-image:
+    name: Build and push
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=ref,event=tag,enable=${{ startsWith(github.ref, 'refs/tags/v') }}
+            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }}
+            type=raw,value=test,enable=true
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args: |
+            APP_VERSION=${{ github.ref_name }}
+
+  deploy-testing:
+    name: Deploy Testing
+    needs: build-and-push-image
+    runs-on: [ self-hosted, Linux ]
+    environment:
+      name: Testing
+      url: https://api.test.profcomff.com/userdata
+    env:
+      CONTAINER_NAME: com_profcomff_api_userdata_test
+    permissions:
+      packages: read
+
+    steps:
+      - name: Pull new version
+        run: docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:test
+
+      - name: Migrate DB
+        run: |
+          docker run \
+            --rm \
+            --network=web \
+            --env DB_DSN=${{ secrets.DB_DSN }} \
+            --name ${{ env.CONTAINER_NAME }}_migration \
+            --workdir="/" \
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:test \
+            alembic upgrade head
+
+      - name: Run new version
+        id: run_prod
+        run: |
+          docker stop ${{ env.CONTAINER_NAME }} || true && docker rm ${{ env.CONTAINER_NAME }} || true
+          docker run \
+            --detach \
+            --restart always \
+            --network=web \
+            --env DB_DSN='${{ secrets.DB_DSN }}' \
+            --env ROOT_PATH='/userdata' \
+            --env GUNICORN_CMD_ARGS='--log-config logging_test.conf' \
+            --name ${{ env.CONTAINER_NAME }} \
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:test
+
+  deploy-production:
+    name: Deploy Production
+    needs: build-and-push-image
+    if: startsWith(github.ref, 'refs/tags/v')
+    runs-on: [ self-hosted, Linux ]
+    environment:
+      name: Production
+      url: https://api.profcomff.com/userdata
+    env:
+      CONTAINER_NAME: com_profcomff_api_userdata
+    permissions:
+      packages: read
+
+    steps:
+      - name: Pull new version
+        run: docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+
+      - name: Migrate DB
+        run: |
+          docker run \
+            --rm \
+            --network=web \
+            --env DB_DSN=${{ secrets.DB_DSN }} \
+            --name ${{ env.CONTAINER_NAME }}_migration \
+            --workdir="/" \
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest \
+            alembic upgrade head
+
+      - name: Run new version
+        id: run_test
+        run: |
+          docker stop ${{ env.CONTAINER_NAME }} || true && docker rm ${{ env.CONTAINER_NAME }} || true
+          docker run \
+            --detach \
+            --restart always \
+            --network=web \
+            --env DB_DSN='${{ secrets.DB_DSN }}' \
+            --env ROOT_PATH='/userdata' \
+            --env GUNICORN_CMD_ARGS='--log-config  logging_prod.conf' \
+            --name ${{ env.CONTAINER_NAME }} \
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest

.github/workflows/tests.yml

@@ -0,0 +1,69 @@
+name: Python package
+
+on:
+  pull_request:
+
+
+jobs:
+  test:
+    name: Unit tests
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up docker
+        uses: docker-practice/actions-setup-docker@master
+      - name: Run postgres
+        run: |
+          docker run -d -p 5432:5432 -e POSTGRES_HOST_AUTH_METHOD=trust --name db-test postgres:15-alpine
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+      - name: Install dependencies
+        run: |
+          python -m ensurepip
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt -r requirements.dev.txt
+      - name: Migrate DB
+        run: |
+          DB_DSN=postgresql://postgres@localhost:5432/postgres alembic upgrade head
+      - name: Build coverage file
+        run: |
+          DB_DSN=postgresql://postgres@localhost:5432/postgres pytest --junitxml=pytest.xml --cov-report=term-missing:skip-covered --cov=userdata_api tests/ | tee pytest-coverage.txt
+      - name: Print report
+        if: always()
+        run: |
+          cat pytest-coverage.txt
+      - name: Pytest coverage comment
+        uses: MishaKav/pytest-coverage-comment@main
+        with:
+          pytest-coverage-path: ./pytest-coverage.txt
+          title: Coverage Report
+          badge-title: Code Coverage
+          hide-badge: false
+          hide-report: false
+          create-new-comment: false
+          hide-comment: false
+          report-only-changed-files: false
+          remove-link-from-badge: false
+          unique-id-for-comment: python3.11
+          junitxml-path: ./pytest.xml
+          junitxml-title: Summary
+  linting:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+      - uses: isort/isort-action@master
+        with:
+          requirementsFiles: "requirements.txt requirements.dev.txt"
+      - uses: psf/black@stable
+      - name: Comment if linting failed
+        if: ${{ failure() }}
+        uses: thollander/actions-comment-pull-request@v2
+        with:
+          message: |
+            :poop: Code linting failed, use `black` and `isort` to fix it.

Makefile

@@ -12,6 +12,15 @@ format:
 	isort ./userdata_api
 	black ./userdata_api
 
+dev-format: format
+	autoflake -r --in-place --remove-all-unused-imports ./tests
+	isort ./tests
+	black ./tests
+	autoflake -r --in-place --remove-all-unused-imports ./migrations
+	isort ./migrations
+	black ./migrations
+
+
 db:
 	docker run -d -p 5432:5432 -e POSTGRES_HOST_AUTH_METHOD=trust --name db-userdata_api postgres:15
 

migrations/env.py

@@ -1,12 +1,12 @@
 from logging.config import fileConfig
 
-from sqlalchemy import engine_from_config
-from sqlalchemy import pool
-
 from alembic import context
+from sqlalchemy import engine_from_config, pool
+
 from userdata_api.models.base import Base
 from userdata_api.settings import get_settings
 
+
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
 config = context.config

migrations/versions/f8c57101c0f6_init.py

@@ -0,0 +1,84 @@
+"""Init
+
+Revision ID: f8c57101c0f6
+Revises: 
+Create Date: 2023-05-09 12:48:25.550608
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+
+revision = 'f8c57101c0f6'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        'category',
+        sa.Column('name', sa.String(), nullable=False),
+        sa.Column('read_scope', sa.String(), nullable=True),
+        sa.Column('update_scope', sa.String(), nullable=True),
+        sa.Column('create_ts', sa.DateTime(), nullable=False),
+        sa.Column('modify_ts', sa.DateTime(), nullable=False),
+        sa.Column('is_deleted', sa.Boolean(), nullable=False),
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.PrimaryKeyConstraint('id'),
+    )
+    op.create_table(
+        'source',
+        sa.Column('name', sa.String(), nullable=False),
+        sa.Column('trust_level', sa.Integer(), nullable=False),
+        sa.Column('create_ts', sa.DateTime(), nullable=False),
+        sa.Column('modify_ts', sa.DateTime(), nullable=False),
+        sa.Column('is_deleted', sa.Boolean(), nullable=False),
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.PrimaryKeyConstraint('id'),
+        sa.UniqueConstraint('name'),
+    )
+    op.create_table(
+        'param',
+        sa.Column('name', sa.String(), nullable=False),
+        sa.Column('category_id', sa.Integer(), nullable=False),
+        sa.Column('is_required', sa.Boolean(), nullable=False),
+        sa.Column('changeable', sa.Boolean(), nullable=False),
+        sa.Column('type', sa.Enum('ALL', 'LAST', 'MOST_TRUSTED', name='viewtype', native_enum=False), nullable=False),
+        sa.Column('create_ts', sa.DateTime(), nullable=False),
+        sa.Column('modify_ts', sa.DateTime(), nullable=False),
+        sa.Column('is_deleted', sa.Boolean(), nullable=False),
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ['category_id'],
+            ['category.id'],
+        ),
+        sa.PrimaryKeyConstraint('id'),
+    )
+    op.create_table(
+        'info',
+        sa.Column('param_id', sa.Integer(), nullable=False),
+        sa.Column('source_id', sa.Integer(), nullable=False),
+        sa.Column('owner_id', sa.Integer(), nullable=False),
+        sa.Column('value', sa.String(), nullable=False),
+        sa.Column('create_ts', sa.DateTime(), nullable=False),
+        sa.Column('modify_ts', sa.DateTime(), nullable=False),
+        sa.Column('is_deleted', sa.Boolean(), nullable=False),
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ['param_id'],
+            ['param.id'],
+        ),
+        sa.ForeignKeyConstraint(
+            ['source_id'],
+            ['source.id'],
+        ),
+        sa.PrimaryKeyConstraint('id'),
+    )
+
+
+def downgrade():
+    op.drop_table('info')
+    op.drop_table('param')
+    op.drop_table('source')
+    op.drop_table('category')

requirements.dev.txt

@@ -5,3 +5,4 @@ isort
 pytest
 pytest-cov
 pytest-mock
+auth-lib-profcomff[testing]