feat: deprecate password files in favor of secrets (#152)

* refactor(jans-pycloudlib): rewrite password file handler * chore: change warning message about password file * refactor(jans-pycloudlib): simplify password file check * feat(configurator): ensure persistence password saved on first init * docs(configurator): add missing initialization attributes * ci: update pycloud dev build * ci: use base ref as base * ci: simplify janspycloud image build * ci: fix branch name reference * ci: update pycloud workflow * ci: fix syntax * ci: fix image build * ci: pull * ci: pull * ci: try force * chore(jans-pycloudlib): updated build Signed-off-by: mo-auto <54212639+mo-auto@users.noreply.github.com> * chore(jans-pycloudlib): updated build Signed-off-by: mo-auto <54212639+mo-auto@users.noreply.github.com> * test(jans-pycloudlib): testcases for passwords * ci: pass token for user to make authenticated call * ci: remove force pushing * ci: change workflow name * ci: passing a token for authentication * ci: add remote url * ci: fix syntax * ci: action push * ci: adjust workflows for creating dev images * ci: enable auto merge * chore(jans-pycloudlib): updated build (#156) Signed-off-by: mo-auto <54212639+mo-auto@users.noreply.github.com> * ci: don't trigger updates on bot pushes * ci: fix syntax * ci: fix syntax * chore(jans-pycloudlib): updated build (#158) Signed-off-by: mo-auto <54212639+mo-auto@users.noreply.github.com> * ci: update build dates in prep for next release * chore(jans-pycloudlib): updated build (#163) Signed-off-by: mo-auto <54212639+mo-auto@users.noreply.github.com> Co-authored-by: Mohammad Abudayyeh <47318409+moabu@users.noreply.github.com> Co-authored-by: mo-auto <54212639+mo-auto@users.noreply.github.com>
JanssenProject · Dec 23, 2021 · f415213 · f415213
1 parent a99bef8
commit f415213
Show file tree

Hide file tree

Showing 30 changed files with 178 additions and 141 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -5,4 +5,6 @@
 # the repo. Unless a later match takes precedence
 
 * @moabu
-/.github/ @moabu
+/.github/ @moabu
+
+/docker-jans-*/requirments.txt @mo-auto
diff --git a/.github/workflows/central_code_quality_check.yml b/.github/workflows/central_code_quality_check.yml
@@ -8,6 +8,7 @@ on:
     branches:
       - master
       - main
+      - "!update-pycloud-in-**"
     paths:
       - "!docker-jans-**/CHANGELOG.md"
       - "!docker-jans-**/version.txt"
@@ -18,6 +19,7 @@ on:
     branches:
       - master
       - main
+      - "!update-pycloud-in-**"
     paths:
       - "!docker-jans-**/CHANGELOG.md"
       - "!docker-jans-**/version.txt"

diff --git a/.github/workflows/commit-check.yml b/.github/workflows/commit-check.yml
@@ -3,6 +3,8 @@
 name: 'Commit Message Check'
 on:
   pull_request:
+    branches-ignore:
+      - "update-pycloud-in-**"
     types:
       - opened
       - edited
@@ -14,6 +16,8 @@ on:
       - "!jans-pycloudlib/CHANGELOG.md"
       - "!jans-pycloudlib/jans/pycloudlib/version.py"
   pull_request_target:
+    branches-ignore:
+      - "update-pycloud-in-**"
     types:
       - opened
       - edited
@@ -25,6 +29,8 @@ on:
       - "!jans-pycloudlib/CHANGELOG.md"
       - "!jans-pycloudlib/jans/pycloudlib/version.py"
   push:
+    branches-ignore:
+      - "update-pycloud-in-**"
     paths:
       - "!docker-jans-**/CHANGELOG.md"
       - "!docker-jans-**/version.txt"

diff --git a/.github/workflows/docker_build_image.yml b/.github/workflows/docker_build_image.yml
@@ -21,6 +21,10 @@ on:
       - "!docker-jans-**/CHANGELOG.md"
       - "!docker-jans-**/version.txt"
       - "!**.md"
+  pull_request_target:
+    branches:
+      # below protected branches prefix only allows internal teams to push to it
+      - cn-*
   workflow_dispatch:
     inputs:
       services:

diff --git a/.github/workflows/docker_imagescan.yml b/.github/workflows/docker_imagescan.yml
@@ -6,13 +6,15 @@ on:
     branches:
       - master
       - main
+      - "!update-pycloud-in-**"
     paths:
       - "docker-jans-**/**"
       - "!**.md"
   pull_request:
     branches:
       - master
       - main
+      - "!update-pycloud-in-**"
     paths:
       - "docker-jans-**/**"
       - "!docker-jans-**/CHANGELOG.md"

diff --git a/.github/workflows/jans_pycloud_build_package.yml b/.github/workflows/jans_pycloud_build_package.yml
@@ -20,98 +20,93 @@ on:
   workflow_dispatch:
 
 jobs:
-  docker:
+  pycloud-updater:
     runs-on: ubuntu-latest
     env:
-      PR_DOCKER_DEV_BRANCH_NAME: update-dev-jans-pycloudlib
-      PR_DOCKER_STABLE_BRANCH_NAME: update-stable-jans-pycloudlib
-      GITHUB_TOKEN: ${{ secrets.MOWORKFLOWTOKEN }}
+      PR_DOCKER_BRANCH_NAME: update-jans-pycloudlib
     steps:
       - name: Checkout
+        if: github.actor != 'mo-auto'
         uses: actions/checkout@v2
 
+      - uses: actions/checkout@v2
+        if: github.event_name == 'pull_request' && github.actor != 'mo-auto'
+        with:
+          ref: ${{ github.head_ref }}
+
       - name: Import GPG key
         id: import_gpg
+        if: github.actor != 'mo-auto'
         uses: crazy-max/ghaction-import-gpg@v4
         with:
           gpg_private_key: ${{ secrets.MOAUTO_GPG_PRIVATE_KEY }}
           passphrase: ${{ secrets.MOAUTO_GPG_PRIVATE_KEY_PASSPHRASE }}
           git_user_signingkey: true
           git_commit_gpgsign: true
 
-      - name: Update dev requriments in docker images
-        if: github.event_name == 'pull_request'
-        id: build_dev_reqs
-        run: |
-          dockerimages="auth-server certmanager client-api config-api configurator fido2 persistence-loader scim"
-          for image in $dockerimages; do
-            sed -i '/git+https/c\git+https://github.com/${{ github.repository }}@${{ env.PR_DOCKER_DEV_BRANCH_NAME }}#egg=jans-pycloudlib&subdirectory=jans-pycloudlib' ./docker-jans-$image/requirements.txt
-          done
-
-      - name: Configure Git and open dev PR
-        if: github.event_name == 'pull_request'
-        run: |
-          git config user.name "mo-auto"
-          git config user.email "54212639+mo-auto@users.noreply.github.com"
-          git config --global user.signingkey "${{ steps.import_gpg.outputs.keyid }}"
-          git add -A
-          git commit -S -s -m "chore(jans-pycloudlib): updated dev build"
-
-
       - name: Update stable requriments in docker images
-        if: github.event_name != 'pull_request'
+        if: github.actor != 'mo-auto'
         id: build_stable_reqs
         run: |
           dockerimages="auth-server certmanager client-api config-api configurator fido2 persistence-loader scim"
           for image in $dockerimages; do
             sed -i '/git+https/c\git+https://github.com/${{ github.repository }}@${{ github.sha }}#egg=jans-pycloudlib&subdirectory=jans-pycloudlib' ./docker-jans-$image/requirements.txt
           done
 
-      - name: Configure Git and open stable PR
-        if: github.event_name != 'pull_request'
+      - name: Configure Git
+        if: github.actor != 'mo-auto'
         run: |
           git config user.name "mo-auto"
           git config user.email "54212639+mo-auto@users.noreply.github.com"
           git config --global user.signingkey "${{ steps.import_gpg.outputs.keyid }}"
           git add -A
-          git commit -S -s -m "chore(jans-pycloudlib): updated stable build"
+          git commit -S -s -m "chore(jans-pycloudlib): updated build"
 
       # Buggy behaviour with gh pr command. Will use the following action until bugs have been fixed.
       #PR=$(gh pr create --head $PR_DOCKER_DEV_BRANCH_NAME --assignee "moabu" --base "master" --body "Updated build date. Auto-generated." --label "enhancement,bot" --reviewer "moabu" --title "chore(Dockerfile): updated build dates" || echo "PR Branch is already open")
       - name: Open PR
-        if: github.event_name != 'pull_request'
+        id: cpr
         uses: peter-evans/create-pull-request@v3
+        if: github.event_name == 'pull_request' && github.actor != 'mo-auto'
         with:
           token: ${{ secrets.MOWORKFLOWTOKEN }}
           committer: mo-auto <54212639+mo-auto@users.noreply.github.com>
           author: mo-auto <54212639+mo-auto@users.noreply.github.com>
-          branch: ${{ env.PR_DOCKER_STABLE_BRANCH_NAME }}
-          title: 'chore(Dockerfiles): updated janspycloud build'
+          branch: 'update-pycloud-in-${{ github.head_ref }}'
+          title: 'chore(Dockerfiles): updated janspycloud build in ${{ github.head_ref }}'
           body: |
-            - Merge PR. Do not leave open
-            - Updated stable build
+            - This PR will automerge
+            - Updated unstable build
             - Auto-generated.
           labels: |
             enhancement
             bot
           assignees: moabu
-          reviewers: moabu
           delete-branch: true
+          base: ${{ github.head_ref }}
+
+      - name: Enable Pull Request Automerge
+        if: steps.cpr.outputs.pull-request-operation == 'created' && github.actor != 'mo-auto'
+        uses: peter-evans/enable-pull-request-automerge@v1
+        with:
+          token: ${{ secrets.MOWORKFLOWTOKEN }}
+          pull-request-number: ${{ steps.cpr.outputs.pull-request-number }}
+          merge-method: squash
 
       # Buggy behaviour with gh pr command. Will use the following action until bugs have been fixed.
       #PR=$(gh pr create --head $PR_DOCKER_DEV_BRANCH_NAME --assignee "moabu" --base "master" --body "Updated build date. Auto-generated." --label "enhancement,bot" --reviewer "moabu" --title "chore(Dockerfile): updated build dates" || echo "PR Branch is already open")
       - name: Open PR
-        if: github.event_name == 'pull_request'
         uses: peter-evans/create-pull-request@v3
+        if: github.event_name != 'pull_request' && github.actor != 'mo-auto'
         with:
           token: ${{ secrets.MOWORKFLOWTOKEN }}
           committer: mo-auto <54212639+mo-auto@users.noreply.github.com>
           author: mo-auto <54212639+mo-auto@users.noreply.github.com>
-          branch: ${{ env.PR_DOCKER_DEV_BRANCH_NAME }}
+          branch: ${{ env.PR_DOCKER_BRANCH_NAME }}
           title: 'chore(Dockerfiles): updated janspycloud build'
           body: |
             - Always leave open
-            - Updated stable build
+            - Updated unstable build
             - Auto-generated.
           labels: |
             enhancement

diff --git a/docker-jans-auth-server/Dockerfile b/docker-jans-auth-server/Dockerfile
@@ -51,7 +51,7 @@ RUN wget -q https://github.com/fabioz/PyDev.Debugger/archive/refs/tags/pydev_deb
 # ===========
 
 ENV CN_VERSION=1.0.0-SNAPSHOT
-ENV CN_BUILD_DATE='2021-12-03 04:06'
+ENV CN_BUILD_DATE='2021-12-23 11:47'
 ENV CN_SOURCE_URL=https://jenkins.jans.io/maven/io/jans/jans-auth-server/${CN_VERSION}/jans-auth-server-${CN_VERSION}.war
 
 # Install Jans Auth

diff --git a/docker-jans-auth-server/requirements.txt b/docker-jans-auth-server/requirements.txt
@@ -1 +1 @@
-git+https://github.com/JanssenProject/jans-cloud-native@eed35d1118df9137989294237623ab4b8e7fac52#egg=jans-pycloudlib&subdirectory=jans-pycloudlib
+git+https://github.com/JanssenProject/jans-cloud-native@20615cf7748184403f8fe71a1465419896f55c96#egg=jans-pycloudlib&subdirectory=jans-pycloudlib
diff --git a/docker-jans-certmanager/Dockerfile b/docker-jans-certmanager/Dockerfile
@@ -16,7 +16,7 @@ RUN apk update \
 
 # JAR files required to generate OpenID Connect keys
 ENV CN_VERSION=1.0.0-SNAPSHOT
-ENV CN_BUILD_DATE='2021-12-03 04:01'
+ENV CN_BUILD_DATE='2021-12-23 11:47'
 ENV CN_SOURCE_URL=https://jenkins.jans.io/maven/io/jans/jans-auth-client/${CN_VERSION}/jans-auth-client-${CN_VERSION}-jar-with-dependencies.jar
 
 RUN wget -q ${CN_SOURCE_URL} -P /app/javalibs/

diff --git a/docker-jans-certmanager/requirements.txt b/docker-jans-certmanager/requirements.txt
@@ -1,2 +1,2 @@
 click==6.7
-git+https://github.com/JanssenProject/jans-cloud-native@eed35d1118df9137989294237623ab4b8e7fac52#egg=jans-pycloudlib&subdirectory=jans-pycloudlib
+git+https://github.com/JanssenProject/jans-cloud-native@20615cf7748184403f8fe71a1465419896f55c96#egg=jans-pycloudlib&subdirectory=jans-pycloudlib
diff --git a/docker-jans-client-api/Dockerfile b/docker-jans-client-api/Dockerfile
@@ -15,7 +15,7 @@ RUN apk update \
 # ==========
 
 ENV CN_VERSION=1.0.0-SNAPSHOT
-ENV CN_BUILD_DATE='2021-10-08 06:18'
+ENV CN_BUILD_DATE='2021-12-21 10:52'
 ENV CN_SOURCE_URL=https://jenkins.jans.io/maven/io/jans/jans-client-api-server/${CN_VERSION}/jans-client-api-server-${CN_VERSION}-distribution.zip
 
 RUN wget -q ${CN_SOURCE_URL} -O /tmp/client-api.zip \

diff --git a/docker-jans-client-api/requirements.txt b/docker-jans-client-api/requirements.txt
@@ -1,2 +1,2 @@
 ruamel.yaml==0.16.10
-git+https://github.com/JanssenProject/jans-cloud-native@eed35d1118df9137989294237623ab4b8e7fac52#egg=jans-pycloudlib&subdirectory=jans-pycloudlib
+git+https://github.com/JanssenProject/jans-cloud-native@20615cf7748184403f8fe71a1465419896f55c96#egg=jans-pycloudlib&subdirectory=jans-pycloudlib
diff --git a/docker-jans-config-api/Dockerfile b/docker-jans-config-api/Dockerfile
@@ -31,7 +31,7 @@ RUN wget -q https://repo1.maven.org/maven2/org/eclipse/jetty/jetty-home/${JETTY_
 # ==========
 
 ENV CN_VERSION=1.0.0-SNAPSHOT
-ENV CN_BUILD_DATE='2021-12-03 04:09'
+ENV CN_BUILD_DATE='2021-12-23 08:48'
 ENV CN_SOURCE_URL=https://jenkins.jans.io/maven/io/jans/jans-config-api-server/${CN_VERSION}/jans-config-api-server-${CN_VERSION}.war
 
 # Install Jans Config API
@@ -51,11 +51,11 @@ EXPOSE 8074
 
 RUN mkdir -p /usr/share/java
 
-ENV SCIM_PLUGIN_BUILD_DATE='2021-12-03 04:09'
+ENV SCIM_PLUGIN_BUILD_DATE='2021-12-23 08:48'
 ENV SCIM_PLUGIN_SOURCE_URL=https://jenkins.jans.io/maven/io/jans/scim-plugin/${CN_VERSION}/scim-plugin-${CN_VERSION}-distribution.jar
 RUN wget -q ${SCIM_PLUGIN_SOURCE_URL} -O /usr/share/java/scim-plugin.jar
 
-ENV ADMIN_UI_PLUGIN_BUILD_DATE='2021-12-03 04:09'
+ENV ADMIN_UI_PLUGIN_BUILD_DATE='2021-12-23 08:48'
 ENV ADMIN_UI_SOURCE_URL=https://maven.jans.io/maven/io/jans/admin-ui-plugin/${CN_VERSION}/admin-ui-plugin-${CN_VERSION}-distribution.jar
 RUN wget -q ${ADMIN_UI_SOURCE_URL} -O /usr/share/java/admin-ui-plugin.jar
 

diff --git a/docker-jans-config-api/requirements.txt b/docker-jans-config-api/requirements.txt
@@ -1 +1 @@
-git+https://github.com/JanssenProject/jans-cloud-native@eed35d1118df9137989294237623ab4b8e7fac52#egg=jans-pycloudlib&subdirectory=jans-pycloudlib
+git+https://github.com/JanssenProject/jans-cloud-native@20615cf7748184403f8fe71a1465419896f55c96#egg=jans-pycloudlib&subdirectory=jans-pycloudlib
diff --git a/docker-jans-configurator/Dockerfile b/docker-jans-configurator/Dockerfile
@@ -16,7 +16,7 @@ RUN apk update \
 
 # JAR files required to generate OpenID Connect keys
 ENV CN_VERSION=1.0.0-SNAPSHOT
-ENV CN_BUILD_DATE='2021-12-03 04:01'
+ENV CN_BUILD_DATE='2021-12-23 11:47'
 ENV CN_SOURCE_URL=https://jenkins.jans.io/maven/io/jans/jans-auth-client/${CN_VERSION}/jans-auth-client-${CN_VERSION}-jar-with-dependencies.jar
 
 RUN wget -q ${CN_SOURCE_URL} -P /app/javalibs/

diff --git a/docker-jans-configurator/README.md b/docker-jans-configurator/README.md
@@ -82,6 +82,11 @@ The load command can be used either to generate or restore config and secret for
 
     - `auth_sig_keys`: space-separated key algorithm for signing (default to `RS256 RS384 RS512 ES256 ES384 ES512 PS256 PS384 PS512`)
     - `auth_enc_keys`: space-separated key algorithm for encryption (default to `RSA1_5 RSA-OAEP`)
+    - `optional_scopes`: list of scopes that will be used (supported scopes are `ldap`, `scim`, `fido2`, `client-api`, `couchbase`, `redis`, `sql`; default to empty list)
+    - `ldap_pw`: user's password to access LDAP database (only used if `optional_scopes` list contains `ldap` scope)
+    - `sql_pw`: user's password to access SQL database (only used if `optional_scopes` list contains `sql` scope)
+    - `couchbase_pw`: user's password to access Couchbase database (only used if `optional_scopes` list contains `couchbase` scope)
+    - `couchbase_superuser_pw`: superuser's password to access Couchbase database (only used if `optional_scopes` list contains `couchbase` scope)
 
 1. Mount the volume into container:
 

diff --git a/docker-jans-configurator/requirements.txt b/docker-jans-configurator/requirements.txt
@@ -1,4 +1,4 @@
 click==6.7
 marshmallow==3.10.0
 fqdn==1.4.0
-git+https://github.com/JanssenProject/jans-cloud-native@eed35d1118df9137989294237623ab4b8e7fac52#egg=jans-pycloudlib&subdirectory=jans-pycloudlib
+git+https://github.com/JanssenProject/jans-cloud-native@20615cf7748184403f8fe71a1465419896f55c96#egg=jans-pycloudlib&subdirectory=jans-pycloudlib
diff --git a/docker-jans-configurator/scripts/bootstrap.py b/docker-jans-configurator/scripts/bootstrap.py
@@ -747,6 +747,8 @@ def couchbase_ctx(self):
         # TODO: move this to persistence-loader?
         self.set_config("couchbaseTrustStoreFn", "/etc/certs/couchbase.pkcs12")
         self.set_secret("couchbase_shib_user_password", get_random_chars)
+        self.set_secret("couchbase_password", self.params["couchbase_pw"])
+        self.set_secret("couchbase_superuser_password", self.params["couchbase_superuser_pw"])
 
     def jackrabbit_ctx(self):
         # self.set_secret("jca_pw", get_random_chars())
@@ -757,6 +759,9 @@ def fido2_ctx(self):
         # TODO: hardcoded in persistence-loader?
         self.set_config("fido2ConfigFolder", "/etc/jans/conf/fido2")
 
+    def sql_ctx(self):
+        self.set_secret("sql_password", self.params["sql_pw"])
+
     def generate(self):
         opt_scopes = self.params["optional_scopes"]
 
@@ -788,6 +793,9 @@ def generate(self):
         if "fido2" in opt_scopes:
             self.fido2_ctx()
 
+        if "sql" in opt_scopes:
+            self.sql_ctx()
+
         # populated config
         return self.ctx
 

diff --git a/docker-jans-configurator/scripts/parameter.py b/docker-jans-configurator/scripts/parameter.py
@@ -30,6 +30,7 @@
     "client-api",
     "couchbase",
     "redis",
+    "sql",
 )
 
 
@@ -54,7 +55,6 @@ class Meta:
 
     email = Email(required=True)
 
-    # see validate_fqdn for validation
     hostname = Str(required=True)
 
     org_name = Str(required=True)
@@ -67,9 +67,18 @@ class Meta:
         missing=[],
     )
 
-    # see validate_ldap_pw for validation
     ldap_pw = Str(missing="", default="")
 
+    sql_pw = Str(missing="", default="")
+
+    couchbase_pw = Str(missing="", default="")
+
+    couchbase_superuser_pw = Str(missing="", default="")
+
+    auth_sig_keys = Str(missing="")
+
+    auth_enc_keys = Str(missing="")
+
     @validates("hostname")
     def validate_fqdn(self, value):
         fqdn = FQDN(value)
@@ -82,7 +91,7 @@ def validate_password(self, value, **kwargs):
             raise ValidationError(
                 "Must be at least 6 characters and include "
                 "one uppercase letter, one lowercase letter, one digit, "
-                " and one special character."
+                "and one special character."
             )
 
     @validates_schema
@@ -93,6 +102,24 @@ def validate_ldap_pw(self, data, **kwargs):
             except ValidationError as exc:
                 raise ValidationError({"ldap_pw": exc.messages})
 
+    @validates_schema
+    def validate_ext_persistence_pw(self, data, **kwargs):
+        err = {}
+        scope_attr_map = [
+            ("sql", "sql_pw"),
+            ("couchbase", "couchbase_pw"),
+        ]
+
+        for scope, attr in scope_attr_map:
+            # note we don't enforce custom password validation as cloud-based
+            # databases may use password that not conform to our policy
+            # hence we simply check for empty password only
+            if scope in data["optional_scopes"] and data[attr] == "":
+                err[attr] = ["Empty password isn't allowed"]
+
+        if err:
+            raise ValidationError(err)
+
 
 def params_from_file(path):
     out = {}