From e0cb5b732f793428a9418b70cab3da5587c669a7 Mon Sep 17 00:00:00 2001 From: Irfan Habib Date: Tue, 22 Aug 2017 13:06:33 +0100 Subject: [PATCH 1/3] Fix unit tests --- .../app-core/backend/mock_server_test.go | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/components/app-core/backend/mock_server_test.go b/components/app-core/backend/mock_server_test.go index b420f0ef33..0eb9eefe2b 100644 --- a/components/app-core/backend/mock_server_test.go +++ b/components/app-core/backend/mock_server_test.go @@ -11,13 +11,12 @@ import ( "testing" "time" - "gopkg.in/DATA-DOG/go-sqlmock.v1" - log "github.com/Sirupsen/logrus" "github.com/gorilla/securecookie" "github.com/gorilla/sessions" "github.com/labstack/echo" "github.com/labstack/echo/engine/standard" + sqlmock "gopkg.in/DATA-DOG/go-sqlmock.v1" "github.com/SUSE/stratos-ui/components/app-core/backend/repository/crypto" "github.com/SUSE/stratos-ui/components/app-core/backend/repository/interfaces" @@ -131,12 +130,11 @@ func setupPortalProxy(db *sql.DB) *portalProxy { urlP, _ := url.Parse("https://login.52.38.188.107.nip.io:50450") pc := interfaces.PortalConfig{ ConsoleConfig: &interfaces.ConsoleConfig{ - ConsoleClient: "console", - ConsoleClientSecret: "", - UAAEndpoint: urlP, - SkipSSLValidation: true, + ConsoleClient: "console", + ConsoleClientSecret: "", + UAAEndpoint: urlP, + SkipSSLValidation: true, ConsoleAdminScope: UAAAdminIdentifier, - }, SessionStoreSecret: "hiddenraisinsohno!", EncryptionKeyInBytes: mockEncryptionKey, @@ -263,21 +261,21 @@ var mockUAAResponse = UAAResponse{ } const ( - mockAPIEndpoint = "https://api.127.0.0.1" - mockAuthEndpoint = "https://login.127.0.0.1" - mockTokenEndpoint = "https://uaa.127.0.0.1" + mockAPIEndpoint = "https://api.127.0.0.1" + mockAuthEndpoint = "https://login.127.0.0.1" + mockTokenEndpoint = "https://uaa.127.0.0.1" mockDopplerEndpoint = "https://doppler.127.0.0.1" - mockProxyVersion = 20161117141922 + mockProxyVersion = 20161117141922 stringCFType = "cf" stringCEType = "hce" selectAnyFromTokens = `SELECT .+ FROM tokens WHERE .+` - insertIntoTokens = `INSERT INTO tokens` - updateTokens = `UPDATE tokens` - selectAnyFromCNSIs = `SELECT (.+) FROM cnsis WHERE (.+)` - insertIntoCNSIs = `INSERT INTO cnsis` - getDbVersion = `SELECT version_id FROM goose_db_version WHERE is_applied = 't' ORDER BY id DESC LIMIT 1` + insertIntoTokens = `INSERT INTO tokens` + updateTokens = `UPDATE tokens` + selectAnyFromCNSIs = `SELECT (.+) FROM cnsis WHERE (.+)` + insertIntoCNSIs = `INSERT INTO cnsis` + getDbVersion = `SELECT version_id FROM goose_db_version WHERE is_applied = '1' ORDER BY id DESC LIMIT 1` ) var rowFieldsForCNSI = []string{"guid", "name", "cnsi_type", "api_endpoint", "auth_endpoint", "token_endpoint", "doppler_logging_endpoint", "skip_ssl_validation"} From 3deff191181b8a040e8eac1c3638a190d35e37ff Mon Sep 17 00:00:00 2001 From: Irfan Habib Date: Tue, 22 Aug 2017 17:57:15 +0100 Subject: [PATCH 2/3] Update helm chart to use mariaDB --- deploy/db/Dockerfile.k8s.postflight-job | 8 +- deploy/db/dbconf.yml | 3 + .../20170818120003_InitialSchema.go | 6 +- .../migrations/20170818162837_SetupSchema.go | 2 +- deploy/db/scripts/run-postflight-job.k8s.sh | 59 +++++++++++---- deploy/docker-compose/build.sh | 4 +- .../console/templates/deployment.yaml | 75 +++++++------------ .../console/templates/post-install.yaml | 28 ++++--- deploy/kubernetes/console/values.yaml | 12 ++- 9 files changed, 103 insertions(+), 94 deletions(-) diff --git a/deploy/db/Dockerfile.k8s.postflight-job b/deploy/db/Dockerfile.k8s.postflight-job index f0f8182468..ab5cc9edf6 100644 --- a/deploy/db/Dockerfile.k8s.postflight-job +++ b/deploy/db/Dockerfile.k8s.postflight-job @@ -1,11 +1,11 @@ -FROM alpine +FROM golang:1.8-alpine RUN apk update && \ - apk add postgresql-client mariadb-client - + apk add postgresql-client mariadb-client git gcc musl-dev +RUN go get bitbucket.org/liamstask/goose/lib/goose +RUN go get github.com/go-sql-driver/mysql COPY goose /usr/local/bin/ COPY deploy/db/dbconf.yml db/dbconf.yml COPY deploy/db/migrations db/migrations COPY deploy/db/scripts/run-postflight-job.k8s.sh /run-postflight-job.sh - CMD ["/run-postflight-job.sh"] diff --git a/deploy/db/dbconf.yml b/deploy/db/dbconf.yml index 4ea1adef9d..8b8ffa1d7a 100644 --- a/deploy/db/dbconf.yml +++ b/deploy/db/dbconf.yml @@ -7,3 +7,6 @@ k8s: mariadb-development: driver: mysql open: stratos:strat0s@tcp(mariadb:3306)/stratos-db?parseTime=true +mariadb-k8s: + driver: mysql + open: $DB_USER:$DB_PASSWORD@tcp($DB_HOST:$DB_PORT)/$DB_DATABASE_NAME?parseTime=true diff --git a/deploy/db/migrations/20170818120003_InitialSchema.go b/deploy/db/migrations/20170818120003_InitialSchema.go index 5a52232fa3..d328b373b2 100644 --- a/deploy/db/migrations/20170818120003_InitialSchema.go +++ b/deploy/db/migrations/20170818120003_InitialSchema.go @@ -22,10 +22,12 @@ func Up_20170818120003(txn *sql.Tx) { createTokens += "auth_token " + binaryDataType + " NOT NULL, " createTokens += "refresh_token " + binaryDataType + " NOT NULL, " createTokens += "token_expiry BIGINT NOT NULL, " - createTokens += "last_updated TIMESTAMP NOT NULL DEFAULT (NOW()) )" + createTokens += "last_updated TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP)" if databaseProvider == "pgsql" { createTokens += " WITH (OIDS=FALSE);" + } else { + createTokens += ";" } _, err := txn.Exec(createTokens) @@ -42,7 +44,7 @@ func Up_20170818120003(txn *sql.Tx) { createCnsisTable += "token_endpoint VARCHAR(255) NOT NULL," createCnsisTable += "doppler_logging_endpoint VARCHAR(255) NOT NULL," createCnsisTable += "skip_ssl_validation BOOLEAN NOT NULL DEFAULT FALSE," - createCnsisTable += "last_updated TIMESTAMP NOT NULL DEFAULT (NOW())," + createCnsisTable += "last_updated TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP," createCnsisTable += "PRIMARY KEY (guid) );" _, err = txn.Exec(createCnsisTable) diff --git a/deploy/db/migrations/20170818162837_SetupSchema.go b/deploy/db/migrations/20170818162837_SetupSchema.go index c54e607ae3..c262b9ec58 100644 --- a/deploy/db/migrations/20170818162837_SetupSchema.go +++ b/deploy/db/migrations/20170818162837_SetupSchema.go @@ -18,7 +18,7 @@ func Up_20170818162837(txn *sql.Tx) { consoleConfigTable += " console_client_secret VARCHAR(255) NOT NULL, " consoleConfigTable += " skip_ssl_validation BOOLEAN NOT NULL DEFAULT FALSE," consoleConfigTable += " is_setup_complete BOOLEAN NOT NULL DEFAULT FALSE," - consoleConfigTable += " last_updated TIMESTAMP NOT NULL DEFAULT (NOW()));" + consoleConfigTable += " last_updated TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP);" _, err := txn.Exec(consoleConfigTable) if err != nil { diff --git a/deploy/db/scripts/run-postflight-job.k8s.sh b/deploy/db/scripts/run-postflight-job.k8s.sh index d7b591ab2c..ce986e8c80 100755 --- a/deploy/db/scripts/run-postflight-job.k8s.sh +++ b/deploy/db/scripts/run-postflight-job.k8s.sh @@ -1,28 +1,53 @@ #!/bin/sh set -e -execStatement() { +function execStatement { stmt=$1 - PGPASSFILE=/tmp/pgpass psql -U $DB_USER -h $DB_HOST -p $DB_PORT -d postgres -w -tc "$stmt" + + if [ "$DATABASE_PROVIDER" = "mysql" ]; then + echo "Executing: mysql -u $DB_ADMIN_USER -h $DB_HOST -P $DB_PORT -p$DB_ADMIN_PASSWORD -e $stmt" + mysql -u $DB_ADMIN_USER -h $DB_HOST -P $DB_PORT -p$DB_ADMIN_PASSWORD -e $stmt + fi + + if [ "$DATABASE_PROVIDER" = "pgsql" ]; then + echo "Executing: PGPASSFILE=/tmp/pgpass psql -U $DB_USER -h $DB_HOST -p $DB_PORT -d postgres -w -tc \"$stmt\"" + PGPASSFILE=/tmp/pgpass psql -U $DB_USER -h $DB_HOST -p $DB_PORT -d postgres -w -tc "$stmt" + fi } -# Save the superuser info to file to ensure secure access -echo "*:$DB_PORT:postgres:$DB_USER:$(cat $DB_PASSWORD_FILE)" > /tmp/pgpass -echo "*:$DB_PORT:$DB_DATABASE_NAME:$DB_USER:$(cat $DB_PASSWORDFILE)" >> /tmp/pgpass -chmod 0600 /tmp/pgpass +if [ "$DATABASE_PROVIDER" = "pgsql" ]; then + # Save the superuser info to file to ensure secure access + echo "*:$DB_PORT:postgres:$DB_USER:$(cat $DB_PASSWORD_FILE)" > /tmp/pgpass + echo "*:$DB_PORT:$DB_DATABASE_NAME:$DB_USER:$(cat $DB_PASSWORDFILE)" >> /tmp/pgpass + chmod 0600 /tmp/pgpass + stratosDbExists=$(execStatement "SELECT 1 FROM pg_database WHERE datname = '$DB_DATABASE_NAME';") + # Get db user password from secrets file + PWD=$(cat $DB_PASSWORDFILE) + DBCONF_KEY=k8s + DB_PASSWORD=$PWD +fi -# Get db user password from secrets file -PWD=$(cat $DB_PASSWORDFILE) +if [ "$DATABASE_PROVIDER" = "mysql" ]; then + echo "DB Provider is MYSQL" + stratosDbExists=$(execStatement "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = '$DB_DATABASE_NAME';") + DBCONF_KEY=mariadb-k8s +fi -# Create the database if necessary -stratosDbExists=$(execStatement "SELECT 1 FROM pg_database WHERE datname = '$DB_DATABASE_NAME';") +# Create DB if neccessary if [ -z "$stratosDbExists" ] ; then echo "Creating database $DB_DATABASE_NAME" execStatement "CREATE DATABASE \"$DB_DATABASE_NAME\";" echo "Creating user $DB_USER" - execStatement "CREATE USER $DB_USER WITH ENCRYPTED PASSWORD '$PWD';" + if [ "$DATABASE_PROVIDER" = "pgsql" ]; then + execStatement "CREATE USER $DB_USER WITH ENCRYPTED PASSWORD '$PWD';" + fi + if [ "$DATABASE_PROVIDER" = "mysql" ]; then + execStatement "CREATE USER $DB_USER IDENTIFIED BY '$DB_PASSWORD';" + fi + echo "Granting privs for $DB_DATABASE_NAME to $DB_USER" execStatement "GRANT ALL PRIVILEGES ON DATABASE \"$DB_DATABASE_NAME\" TO $DB_USER;" + DBCONF_KEY=mariadb-k8s else echo "$DB_DATABASE_NAME already exists" fi @@ -30,25 +55,27 @@ fi # Migrate the database if necessary echo "Checking database to see if migration is necessary." +echo "DBCONFIG: $DBCONF_KEY" +echo "Connection string: $DB_USER:$DB_PASSWORD@tcp($DB_HOST:$DB_PORT)/$DB_DATABASE_NAME?parseTime=true" # Check the version echo "Checking database version." -DB_PASSWORD=$PWD goose --env=k8s dbversion +goose --env=$DBCONF_KEY dbversion # Check the status echo "Checking database status." -DB_PASSWORD=$PWD goose --env=k8s status +goose --env=$DBCONF_KEY status # Run migrations echo "Attempting database migrations." -DB_PASSWORD=$PWD goose --env=k8s up +goose --env=$DBCONF_KEY up # CHeck the status echo "Checking database status." -DB_PASSWORD=$PWD goose --env=k8s status +goose --env=$DBCONF_KEY status # Check the version echo "Checking database version." -DB_PASSWORD=$PWD goose --env=k8s dbversion +goose --env=$DBCONF_KEY dbversion echo "Database operation(s) complete." diff --git a/deploy/docker-compose/build.sh b/deploy/docker-compose/build.sh index 084bf4d96a..2990b9d35d 100755 --- a/deploy/docker-compose/build.sh +++ b/deploy/docker-compose/build.sh @@ -238,8 +238,8 @@ cleanup updateTagForRelease # Build all of the components that make up the Console -#buildProxy -#buildGoose +buildProxy +buildGoose buildUI # Done diff --git a/deploy/kubernetes/console/templates/deployment.yaml b/deploy/kubernetes/console/templates/deployment.yaml index 1ef2e09e70..2a65935b71 100644 --- a/deploy/kubernetes/console/templates/deployment.yaml +++ b/deploy/kubernetes/console/templates/deployment.yaml @@ -1,24 +1,5 @@ --- apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: "{{ .Release.Name }}-postgres-volume" - labels: - app: "{{ .Release.Name }}" - annotations: - {{- if .Values.storageClass }} - volume.beta.kubernetes.io/storage-class: {{ .Values.storageClass | quote }} - {{- else }} - volume.alpha.kubernetes.io/storage-class: default - {{- end }} -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 2Gi ---- -apiVersion: v1 kind: Secret type: Opaque metadata: @@ -27,7 +8,7 @@ metadata: app: "{{ .Release.Name }}" data: stolon: {{ .Values.dbPassword | b64enc }} - db-password: {{ .Values.dbPassword | b64enc }} + db-password: {{ .Values.mariadb.mariadbPassword | b64enc }} console-cert-key: {{ .Files.Get "ssl/console.key" | b64enc }} console-cert: {{ .Files.Get "ssl/console.crt" | b64enc }} @@ -64,23 +45,21 @@ spec: name: "{{ .Release.Name }}-postflight" env: - name: DB_HOST - value: "{{ .Release.Name }}-postgres-int" + value: "{{ .Release.Name }}-mariadb" - name: DB_PORT - value: "5432" - - name: POSTGRES_USER - value: postgres - - name: POSTGRES_PASSWORD_FILE - value: /etc/secrets/stolon - - name: DB_DATABASE_NAME - value: console-db + value: "3306" + - name: DB_ADMIN_USER + value: "{{ .Values.mariadb.adminUser }}" + - name: DB_ADMIN_PASSWORD + value: "{{ .Values.mariadb.mariadbRootPassword }}" + - name: DATABASE_PROVIDER + value: "{{ .Values.dbProvider }}" + - name: DB_PASSWORD + value: "{{ .Values.mariadb.mariadbPassword }}" - name: DB_USER - value: console - - name: DB_PASSWORDFILE - value: /etc/secrets/db-password - - name: DB_SSL_MODE - value: disable - - name: PGCONNECT_TIMEOUT - value: "10" + value: "{{ .Values.mariadb.mariadbUser }}" + - name: DB_DATABASE_NAME + value: "{{ .Values.mariadb.mariadbDatabase }}" - name: DO_NOT_QUIT value: "true" - name: UPGRADE_VOLUME @@ -122,22 +101,21 @@ spec: {{- end }} env: - name: DB_USER - value: console + value: "{{ .Values.mariadb.mariadbUser }}" - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: "{{ .Release.Name }}-secret" - key: db-password + value: "{{ .Values.mariadb.mariadbPassword }}" - name: DB_DATABASE_NAME - value: console-db + value: "{{ .Values.mariadb.mariadbDatabase }}" - name: DB_HOST - value: "{{ .Release.Name }}-postgres-int" + value: "{{ .Release.Name }}-mariadb" - name: DB_PORT - value: "5432" - - name: DB_CONNECT_TIMEOUT_IN_SECS - value: "5" - - name: DB_SSL_MODE - value: disable + value: "3306" + - name: DATABASE_PROVIDER + value: "{{ .Values.dbProvider }}" + - name: DB_ADMIN_USER + value: "{{ .Values.mariadb.adminUser }}" + - name: DB_ADMIN_PASSWORD + value: "{{ .Values.mariadb.mariadbRootPassword }}" - name: HTTP_CONNECTION_TIMEOUT_IN_SECS value: "10" - name: HTTP_CLIENT_TIMEOUT_IN_SECS @@ -191,9 +169,6 @@ spec: name: "{{ .Release.Name }}-secret" readOnly: true volumes: - - name: "{{ .Release.Name }}-postgres-volume" - persistentVolumeClaim: - claimName: "{{ .Release.Name }}-postgres-volume" - name: "{{ .Release.Name }}-upgrade-volume" persistentVolumeClaim: claimName: "{{ .Release.Name }}-upgrade-volume" diff --git a/deploy/kubernetes/console/templates/post-install.yaml b/deploy/kubernetes/console/templates/post-install.yaml index c3b4701d97..900e749b56 100644 --- a/deploy/kubernetes/console/templates/post-install.yaml +++ b/deploy/kubernetes/console/templates/post-install.yaml @@ -19,23 +19,21 @@ spec: containers: - env: - name: DB_HOST - value: "{{ .Release.Name }}-postgres-int" + value: "{{ .Release.Name }}-mariadb" - name: DB_PORT - value: "5432" - - name: POSTGRES_USER - value: postgres - - name: POSTGRES_PASSWORD_FILE - value: /etc/secrets/stolon - - name: DB_DATABASE_NAME - value: console-db + value: "3306" + - name: DB_ADMIN_USER + value: "{{ .Values.mariadb.adminUser }}" + - name: DB_ADMIN_PASSWORD + value: "{{ .Values.mariadb.mariadbRootPassword }}" + - name: DATABASE_PROVIDER + value: "{{ .Values.dbProvider }}" + - name: DB_PASSWORD + value: "{{ .Values.mariadb.mariadbPassword }}" - name: DB_USER - value: console - - name: DB_PASSWORDFILE - value: /etc/secrets/pgsql-password - - name: DB_SSL_MODE - value: disable - - name: PGCONNECT_TIMEOUT - value: "10" + value: "{{ .Values.mariadb.mariadbUser }}" + - name: DB_DATABASE_NAME + value: "{{ .Values.mariadb.mariadbDatabase }}" - name: UPGRADE_VOLUME value: "{{ .Release.Name }}-upgrade-volume" - name: UPGRADE_LOCK_FILENAME diff --git a/deploy/kubernetes/console/values.yaml b/deploy/kubernetes/console/values.yaml index 3afbffc89f..14ccd1c243 100644 --- a/deploy/kubernetes/console/values.yaml +++ b/deploy/kubernetes/console/values.yaml @@ -6,6 +6,7 @@ dockerOrg: splatform dockerRegistry: docker.io # Specify default DB password dbPassword: changeme +dbProvider: mysql # Provide Proxy settings if required #httpProxy: proxy.corp.net #httpsProxy: proxy.corp.net @@ -24,12 +25,15 @@ images: #storageClass: default # MariaDB chart configuration mariadb: - usePassword: true + # Only required for creating the databases mariadbRootPassword: changeme + adminUser: root + # Credentials for user + mariadbUser: console + mariadbPassword: changeme + mariadbDatabase: console persistence: - enabled: true - existingClaim: true - accessMode: ReadWrite + accessMode: ReadWriteMany size: 1Gi # storageClass: default metrics: From 0ac8d7a7856e4ba6faca874e7d7c20d669dd3639 Mon Sep 17 00:00:00 2001 From: Irfan Habib Date: Fri, 8 Sep 2017 15:05:38 +0100 Subject: [PATCH 3/3] Updated post-flight script in response to feedback --- deploy/db/scripts/run-postflight-job.k8s.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/deploy/db/scripts/run-postflight-job.k8s.sh b/deploy/db/scripts/run-postflight-job.k8s.sh index ce986e8c80..1f59e7e17d 100755 --- a/deploy/db/scripts/run-postflight-job.k8s.sh +++ b/deploy/db/scripts/run-postflight-job.k8s.sh @@ -22,9 +22,8 @@ if [ "$DATABASE_PROVIDER" = "pgsql" ]; then chmod 0600 /tmp/pgpass stratosDbExists=$(execStatement "SELECT 1 FROM pg_database WHERE datname = '$DB_DATABASE_NAME';") # Get db user password from secrets file - PWD=$(cat $DB_PASSWORDFILE) + DB_PASSWORD=$(cat $DB_PASSWORDFILE) DBCONF_KEY=k8s - DB_PASSWORD=$PWD fi if [ "$DATABASE_PROVIDER" = "mysql" ]; then @@ -39,7 +38,7 @@ if [ -z "$stratosDbExists" ] ; then execStatement "CREATE DATABASE \"$DB_DATABASE_NAME\";" echo "Creating user $DB_USER" if [ "$DATABASE_PROVIDER" = "pgsql" ]; then - execStatement "CREATE USER $DB_USER WITH ENCRYPTED PASSWORD '$PWD';" + execStatement "CREATE USER $DB_USER WITH ENCRYPTED PASSWORD '$DB_PASSWORD';" fi if [ "$DATABASE_PROVIDER" = "mysql" ]; then execStatement "CREATE USER $DB_USER IDENTIFIED BY '$DB_PASSWORD';" @@ -47,7 +46,6 @@ if [ -z "$stratosDbExists" ] ; then echo "Granting privs for $DB_DATABASE_NAME to $DB_USER" execStatement "GRANT ALL PRIVILEGES ON DATABASE \"$DB_DATABASE_NAME\" TO $DB_USER;" - DBCONF_KEY=mariadb-k8s else echo "$DB_DATABASE_NAME already exists" fi