diff --git a/deploy/db/Dockerfile.k8s.postflight-job b/deploy/db/Dockerfile.k8s.postflight-job index d2caffe43d..ab5cc9edf6 100644 --- a/deploy/db/Dockerfile.k8s.postflight-job +++ b/deploy/db/Dockerfile.k8s.postflight-job @@ -1,11 +1,11 @@ -FROM alpine +FROM golang:1.8-alpine RUN apk update && \ - apk add postgresql-client - + apk add postgresql-client mariadb-client git gcc musl-dev +RUN go get bitbucket.org/liamstask/goose/lib/goose +RUN go get github.com/go-sql-driver/mysql COPY goose /usr/local/bin/ COPY deploy/db/dbconf.yml db/dbconf.yml COPY deploy/db/migrations db/migrations COPY deploy/db/scripts/run-postflight-job.k8s.sh /run-postflight-job.sh - CMD ["/run-postflight-job.sh"] diff --git a/deploy/db/dbconf.yml b/deploy/db/dbconf.yml index 4ea1adef9d..8b8ffa1d7a 100644 --- a/deploy/db/dbconf.yml +++ b/deploy/db/dbconf.yml @@ -7,3 +7,6 @@ k8s: mariadb-development: driver: mysql open: stratos:strat0s@tcp(mariadb:3306)/stratos-db?parseTime=true +mariadb-k8s: + driver: mysql + open: $DB_USER:$DB_PASSWORD@tcp($DB_HOST:$DB_PORT)/$DB_DATABASE_NAME?parseTime=true diff --git a/deploy/db/migrations/20170818120003_InitialSchema.go b/deploy/db/migrations/20170818120003_InitialSchema.go index 591a108309..f341431712 100644 --- a/deploy/db/migrations/20170818120003_InitialSchema.go +++ b/deploy/db/migrations/20170818120003_InitialSchema.go @@ -22,10 +22,12 @@ func Up_20170818120003(txn *sql.Tx) { createTokens += "auth_token " + binaryDataType + " NOT NULL, " createTokens += "refresh_token " + binaryDataType + " NOT NULL, " createTokens += "token_expiry BIGINT NOT NULL, " - createTokens += "last_updated TIMESTAMP NOT NULL DEFAULT (NOW()) )" + createTokens += "last_updated TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP)" if databaseProvider == "pgsql" { createTokens += " WITH (OIDS=FALSE);" + } else { + createTokens += ";" } _, err := txn.Exec(createTokens) @@ -42,7 +44,7 @@ func Up_20170818120003(txn *sql.Tx) { createCnsisTable += "token_endpoint VARCHAR(255) NOT NULL," createCnsisTable += "doppler_logging_endpoint VARCHAR(255) NOT NULL," createCnsisTable += "skip_ssl_validation BOOLEAN NOT NULL DEFAULT FALSE," - createCnsisTable += "last_updated TIMESTAMP NOT NULL DEFAULT (NOW())," + createCnsisTable += "last_updated TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP," createCnsisTable += "PRIMARY KEY (guid) );" _, err = txn.Exec(createCnsisTable) diff --git a/deploy/db/migrations/20170818162837_SetupSchema.go b/deploy/db/migrations/20170818162837_SetupSchema.go index e40d66484b..8bffe30b19 100644 --- a/deploy/db/migrations/20170818162837_SetupSchema.go +++ b/deploy/db/migrations/20170818162837_SetupSchema.go @@ -18,7 +18,7 @@ func Up_20170818162837(txn *sql.Tx) { consoleConfigTable += " console_client_secret VARCHAR(255) NOT NULL, " consoleConfigTable += " skip_ssl_validation BOOLEAN NOT NULL DEFAULT FALSE," consoleConfigTable += " is_setup_complete BOOLEAN NOT NULL DEFAULT FALSE," - consoleConfigTable += " last_updated TIMESTAMP NOT NULL DEFAULT (NOW()));" + consoleConfigTable += " last_updated TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP);" _, err := txn.Exec(consoleConfigTable) if err != nil { diff --git a/deploy/db/scripts/run-postflight-job.k8s.sh b/deploy/db/scripts/run-postflight-job.k8s.sh index aeca493027..1f59e7e17d 100755 --- a/deploy/db/scripts/run-postflight-job.k8s.sh +++ b/deploy/db/scripts/run-postflight-job.k8s.sh @@ -1,66 +1,79 @@ #!/bin/sh set -e -execStatement() { +function execStatement { stmt=$1 - PGPASSFILE=/tmp/pgpass psql -U $POSTGRES_USER -h $PGSQL_HOST -p $PGSQL_PORT -d postgres -w -tc "$stmt" -} -execBackupRestore() { - orig_server="hsc-stproxy-int" - dest_server=$PGSQL_HOST - bkup="pg_dump -U $PGSQL_USER -h $orig_server -p $PGSQL_PORT -w $PGSQL_DATABASE" - stor="psql -U $PGSQL_USER -h $dest_server -p $PGSQL_PORT -w $PGSQL_DATABASE" + if [ "$DATABASE_PROVIDER" = "mysql" ]; then + echo "Executing: mysql -u $DB_ADMIN_USER -h $DB_HOST -P $DB_PORT -p$DB_ADMIN_PASSWORD -e $stmt" + mysql -u $DB_ADMIN_USER -h $DB_HOST -P $DB_PORT -p$DB_ADMIN_PASSWORD -e $stmt + fi - PGPASSFILE=/tmp/pgpass $bkup | PGPASSFILE=/tmp/pgpass $stor + if [ "$DATABASE_PROVIDER" = "pgsql" ]; then + echo "Executing: PGPASSFILE=/tmp/pgpass psql -U $DB_USER -h $DB_HOST -p $DB_PORT -d postgres -w -tc \"$stmt\"" + PGPASSFILE=/tmp/pgpass psql -U $DB_USER -h $DB_HOST -p $DB_PORT -d postgres -w -tc "$stmt" + fi } -# Save the superuser info to file to ensure secure access -echo "*:$PGSQL_PORT:postgres:$POSTGRES_USER:$(cat $POSTGRES_PASSWORD_FILE)" > /tmp/pgpass -echo "*:$PGSQL_PORT:$PGSQL_DATABASE:$PGSQL_USER:$(cat $PGSQL_PASSWORDFILE)" >> /tmp/pgpass -chmod 0600 /tmp/pgpass +if [ "$DATABASE_PROVIDER" = "pgsql" ]; then + # Save the superuser info to file to ensure secure access + echo "*:$DB_PORT:postgres:$DB_USER:$(cat $DB_PASSWORD_FILE)" > /tmp/pgpass + echo "*:$DB_PORT:$DB_DATABASE_NAME:$DB_USER:$(cat $DB_PASSWORDFILE)" >> /tmp/pgpass + chmod 0600 /tmp/pgpass + stratosDbExists=$(execStatement "SELECT 1 FROM pg_database WHERE datname = '$DB_DATABASE_NAME';") + # Get db user password from secrets file + DB_PASSWORD=$(cat $DB_PASSWORDFILE) + DBCONF_KEY=k8s +fi -# Get db user password from secrets file -PWD=$(cat $PGSQL_PASSWORDFILE) +if [ "$DATABASE_PROVIDER" = "mysql" ]; then + echo "DB Provider is MYSQL" + stratosDbExists=$(execStatement "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = '$DB_DATABASE_NAME';") + DBCONF_KEY=mariadb-k8s +fi -# Create the database if necessary -stratosDbExists=$(execStatement "SELECT 1 FROM pg_database WHERE datname = '$PGSQL_DATABASE';") +# Create DB if neccessary if [ -z "$stratosDbExists" ] ; then - echo "Creating database $PGSQL_DATABASE" - execStatement "CREATE DATABASE \"$PGSQL_DATABASE\";" - echo "Creating user $PGSQL_USER" - execStatement "CREATE USER $PGSQL_USER WITH ENCRYPTED PASSWORD '$PWD';" - echo "Granting privs for $PGSQL_DATABASE to $PGSQL_USER" - execStatement "GRANT ALL PRIVILEGES ON DATABASE \"$PGSQL_DATABASE\" TO $PGSQL_USER;" + echo "Creating database $DB_DATABASE_NAME" + execStatement "CREATE DATABASE \"$DB_DATABASE_NAME\";" + echo "Creating user $DB_USER" + if [ "$DATABASE_PROVIDER" = "pgsql" ]; then + execStatement "CREATE USER $DB_USER WITH ENCRYPTED PASSWORD '$DB_PASSWORD';" + fi + if [ "$DATABASE_PROVIDER" = "mysql" ]; then + execStatement "CREATE USER $DB_USER IDENTIFIED BY '$DB_PASSWORD';" + fi + + echo "Granting privs for $DB_DATABASE_NAME to $DB_USER" + execStatement "GRANT ALL PRIVILEGES ON DATABASE \"$DB_DATABASE_NAME\" TO $DB_USER;" else - echo "$PGSQL_DATABASE already exists" + echo "$DB_DATABASE_NAME already exists" fi -# Backup existing database from stolon cluster and restore it to the single instance -#execBackupRestore - # Migrate the database if necessary echo "Checking database to see if migration is necessary." +echo "DBCONFIG: $DBCONF_KEY" +echo "Connection string: $DB_USER:$DB_PASSWORD@tcp($DB_HOST:$DB_PORT)/$DB_DATABASE_NAME?parseTime=true" # Check the version echo "Checking database version." -PGSQL_PASSWORD=$PWD goose --env=k8s dbversion +goose --env=$DBCONF_KEY dbversion # Check the status echo "Checking database status." -PGSQL_PASSWORD=$PWD goose --env=k8s status +goose --env=$DBCONF_KEY status # Run migrations echo "Attempting database migrations." -PGSQL_PASSWORD=$PWD goose --env=k8s up +goose --env=$DBCONF_KEY up # CHeck the status echo "Checking database status." -PGSQL_PASSWORD=$PWD goose --env=k8s status +goose --env=$DBCONF_KEY status # Check the version echo "Checking database version." -PGSQL_PASSWORD=$PWD goose --env=k8s dbversion +goose --env=$DBCONF_KEY dbversion echo "Database operation(s) complete." diff --git a/deploy/docker-compose/build.sh b/deploy/docker-compose/build.sh index 084bf4d96a..2990b9d35d 100755 --- a/deploy/docker-compose/build.sh +++ b/deploy/docker-compose/build.sh @@ -238,8 +238,8 @@ cleanup updateTagForRelease # Build all of the components that make up the Console -#buildProxy -#buildGoose +buildProxy +buildGoose buildUI # Done diff --git a/deploy/kubernetes/console/templates/deployment.yaml b/deploy/kubernetes/console/templates/deployment.yaml index 5005a5c049..d6d88e97d3 100644 --- a/deploy/kubernetes/console/templates/deployment.yaml +++ b/deploy/kubernetes/console/templates/deployment.yaml @@ -1,24 +1,5 @@ --- apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: "{{ .Release.Name }}-postgres-volume" - labels: - app: "{{ .Release.Name }}" - annotations: - {{- if .Values.storageClass }} - volume.beta.kubernetes.io/storage-class: {{ .Values.storageClass | quote }} - {{- else }} - volume.alpha.kubernetes.io/storage-class: default - {{- end }} -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: 2Gi ---- -apiVersion: v1 kind: Secret type: Opaque metadata: @@ -27,7 +8,7 @@ metadata: app: "{{ .Release.Name }}" data: stolon: {{ .Values.dbPassword | b64enc }} - pgsql-password: {{ .Values.dbPassword | b64enc }} + db-password: {{ .Values.mariadb.mariadbPassword | b64enc }} console-cert-key: {{ .Files.Get "ssl/console.key" | b64enc }} console-cert: {{ .Files.Get "ssl/console.crt" | b64enc }} @@ -63,24 +44,22 @@ spec: - image: {{.Values.dockerRegistry}}/{{.Values.dockerOrg}}/{{.Values.images.postflight}}:{{.Values.consoleVersion}} name: "{{ .Release.Name }}-postflight" env: - - name: PGSQL_HOST - value: "{{ .Release.Name }}-postgres-int" - - name: PGSQL_PORT - value: "5432" - - name: POSTGRES_USER - value: postgres - - name: POSTGRES_PASSWORD_FILE - value: /etc/secrets/stolon - - name: PGSQL_DATABASE - value: console-db - - name: PGSQL_USER - value: console - - name: PGSQL_PASSWORDFILE - value: /etc/secrets/pgsql-password - - name: PGSQL_SSL_MODE - value: disable - - name: PGCONNECT_TIMEOUT - value: "10" + - name: DB_HOST + value: "{{ .Release.Name }}-mariadb" + - name: DB_PORT + value: "3306" + - name: DB_ADMIN_USER + value: "{{ .Values.mariadb.adminUser }}" + - name: DB_ADMIN_PASSWORD + value: "{{ .Values.mariadb.mariadbRootPassword }}" + - name: DATABASE_PROVIDER + value: "{{ .Values.dbProvider }}" + - name: DB_PASSWORD + value: "{{ .Values.mariadb.mariadbPassword }}" + - name: DB_USER + value: "{{ .Values.mariadb.mariadbUser }}" + - name: DB_DATABASE_NAME + value: "{{ .Values.mariadb.mariadbDatabase }}" - name: DO_NOT_QUIT value: "true" - name: UPGRADE_VOLUME @@ -121,23 +100,22 @@ spec: name: proxy {{- end }} env: - - name: PGSQL_USER - value: console - - name: PGSQL_PASSWORD - valueFrom: - secretKeyRef: - name: "{{ .Release.Name }}-secret" - key: pgsql-password - - name: PGSQL_DATABASE - value: console-db - - name: PGSQL_HOST - value: "{{ .Release.Name }}-postgres-int" - - name: PGSQL_PORT - value: "5432" - - name: PGSQL_CONNECT_TIMEOUT_IN_SECS - value: "5" - - name: PGSQL_SSL_MODE - value: disable + - name: DB_USER + value: "{{ .Values.mariadb.mariadbUser }}" + - name: DB_PASSWORD + value: "{{ .Values.mariadb.mariadbPassword }}" + - name: DB_DATABASE_NAME + value: "{{ .Values.mariadb.mariadbDatabase }}" + - name: DB_HOST + value: "{{ .Release.Name }}-mariadb" + - name: DB_PORT + value: "3306" + - name: DATABASE_PROVIDER + value: "{{ .Values.dbProvider }}" + - name: DB_ADMIN_USER + value: "{{ .Values.mariadb.adminUser }}" + - name: DB_ADMIN_PASSWORD + value: "{{ .Values.mariadb.mariadbRootPassword }}" - name: HTTP_CONNECTION_TIMEOUT_IN_SECS value: "10" - name: HTTP_CLIENT_TIMEOUT_IN_SECS @@ -191,9 +169,6 @@ spec: name: "{{ .Release.Name }}-secret" readOnly: true volumes: - - name: "{{ .Release.Name }}-postgres-volume" - persistentVolumeClaim: - claimName: "{{ .Release.Name }}-postgres-volume" - name: "{{ .Release.Name }}-upgrade-volume" persistentVolumeClaim: claimName: "{{ .Release.Name }}-upgrade-volume" @@ -203,67 +178,3 @@ spec: - name: "{{ .Release.Name }}-secret" secret: secretName: "{{ .Release.Name }}-secret" ---- -apiVersion: apps/v1beta1 -kind: StatefulSet -metadata: - name: postgres -spec: - serviceName: "postgres" - replicas: 1 - template: - metadata: - labels: - app: "{{ .Release.Name }}" - component: postgres - spec: - containers: - - image: {{.Values.dockerRegistry}}/{{.Values.dockerOrg}}/{{.Values.images.postgres}}:{{.Values.consoleVersion}} - name: postgres - env: - - name: POSTGRES_USER - value: postgres - - name: POSTGRES_PASSWORD_FILE - value: /etc/secrets/stolon - - name: PGDATA - value: /stolon-data - - name: HTTP_PROXY - {{- if .Values.httpProxy }} - value: {{.Values.httpProxy}} - {{- end }} - - name: HTTPS_PROXY - {{- if .Values.httpsProxy }} - value: {{.Values.httpsProxy}} - {{- end }} - - name: NO_PROXY - {{- if .Values.noProxy }} - value: {{.Values.noProxy}} - {{- end }} - - name: FTP_PROXY - {{- if .Values.ftpProxy }} - value: {{.Values.ftpProxy}} - {{- end }} - - name: SOCKS_PROXY - {{- if .Values.socksProxy }} - value: {{.Values.socksProxy}} - {{- end }} - volumeMounts: - - mountPath: /stolon-data - name: "{{ .Release.Name }}-postgres-volume" - - mountPath: /etc/secrets/ - name: "{{ .Release.Name }}-secret" - readOnly: true - ports: - - containerPort: 5432 - name: postgres - protocol: TCP - volumes: - - name: "{{ .Release.Name }}-postgres-volume" - persistentVolumeClaim: - claimName: "{{ .Release.Name }}-postgres-volume" - - name: "{{ .Release.Name }}-encryption-key-volume" - persistentVolumeClaim: - claimName: "{{ .Release.Name }}-encryption-key-volume" - - name: "{{ .Release.Name }}-secret" - secret: - secretName: "{{ .Release.Name }}-secret" diff --git a/deploy/kubernetes/console/templates/post-install.yaml b/deploy/kubernetes/console/templates/post-install.yaml index cdd844aa8f..900e749b56 100644 --- a/deploy/kubernetes/console/templates/post-install.yaml +++ b/deploy/kubernetes/console/templates/post-install.yaml @@ -18,24 +18,22 @@ spec: spec: containers: - env: - - name: PGSQL_HOST - value: "{{ .Release.Name }}-postgres-int" - - name: PGSQL_PORT - value: "5432" - - name: POSTGRES_USER - value: postgres - - name: POSTGRES_PASSWORD_FILE - value: /etc/secrets/stolon - - name: PGSQL_DATABASE - value: console-db - - name: PGSQL_USER - value: console - - name: PGSQL_PASSWORDFILE - value: /etc/secrets/pgsql-password - - name: PGSQL_SSL_MODE - value: disable - - name: PGCONNECT_TIMEOUT - value: "10" + - name: DB_HOST + value: "{{ .Release.Name }}-mariadb" + - name: DB_PORT + value: "3306" + - name: DB_ADMIN_USER + value: "{{ .Values.mariadb.adminUser }}" + - name: DB_ADMIN_PASSWORD + value: "{{ .Values.mariadb.mariadbRootPassword }}" + - name: DATABASE_PROVIDER + value: "{{ .Values.dbProvider }}" + - name: DB_PASSWORD + value: "{{ .Values.mariadb.mariadbPassword }}" + - name: DB_USER + value: "{{ .Values.mariadb.mariadbUser }}" + - name: DB_DATABASE_NAME + value: "{{ .Values.mariadb.mariadbDatabase }}" - name: UPGRADE_VOLUME value: "{{ .Release.Name }}-upgrade-volume" - name: UPGRADE_LOCK_FILENAME diff --git a/deploy/kubernetes/console/values.yaml b/deploy/kubernetes/console/values.yaml index 177e017eaa..14ccd1c243 100644 --- a/deploy/kubernetes/console/values.yaml +++ b/deploy/kubernetes/console/values.yaml @@ -6,6 +6,7 @@ dockerOrg: splatform dockerRegistry: docker.io # Specify default DB password dbPassword: changeme +dbProvider: mysql # Provide Proxy settings if required #httpProxy: proxy.corp.net #httpsProxy: proxy.corp.net @@ -21,4 +22,19 @@ images: preflight: stratos-preflight-job postflight: stratos-postflight-job # Specify which storage class should be used for PVCs -#storageClass: default \ No newline at end of file +#storageClass: default +# MariaDB chart configuration +mariadb: + # Only required for creating the databases + mariadbRootPassword: changeme + adminUser: root + # Credentials for user + mariadbUser: console + mariadbPassword: changeme + mariadbDatabase: console + persistence: + accessMode: ReadWriteMany + size: 1Gi +# storageClass: default + metrics: + enabled: false