From 95fd66e22b8b818b3c29fe852c6e622d8aa19e53 Mon Sep 17 00:00:00 2001 From: Marc Campbell Date: Wed, 5 Jun 2019 23:21:22 +0000 Subject: [PATCH] Foreign key support --- .buildkite/pipeline.yml | 22 +- README.md | 11 +- config/crds/schemas_v1alpha1_table.yaml | 53 ++- config/dev/database/postgres-10.7.yaml | 3 +- config/dev/github/projects-table-pg.yaml | 23 ++ config/dev/github/projects.yaml | 20 ++ config/dev/github/users-table-pg.yaml | 2 - config/dev/github/users.yaml | 15 + integration/.gitignore | 1 + integration/Makefile | 40 +-- integration/README.md | 9 + integration/cmd/integration/main.go | 7 - integration/manifests/manager.yaml | 327 ------------------ integration/pkg/cli/root.go | 43 --- integration/pkg/cli/run.go | 27 -- integration/pkg/runner/cluster.go | 242 ------------- integration/pkg/runner/crd.go | 50 --- integration/pkg/runner/runner.go | 151 -------- integration/pkg/runner/types.go | 53 --- integration/pkg/runner/verify.go | 12 - integration/postgres/Makefile | 25 -- integration/postgres/README.md | 11 - .../tests/mysql/create-table/Dockerfile | 9 + integration/tests/mysql/create-table/Makefile | 55 +++ .../create-table/expect/kustomization.yaml | 3 + .../mysql/create-table/expect/other.yaml | 20 ++ .../mysql/create-table/expect/users.yaml | 24 ++ .../tests/mysql/create-table/fixtures.sql | 4 + .../tests/mysql/create-table/specs/users.yaml | 13 + .../tests/mysql/foreign-key-alter/Dockerfile | 9 + .../tests/mysql/foreign-key-alter/Makefile | 54 +++ .../foreign-key-alter/expect/issues.yaml | 28 ++ .../expect/kustomization.yaml | 4 + .../foreign-key-alter/expect/projects.yaml | 20 ++ .../mysql/foreign-key-alter/expect/users.yaml | 20 ++ .../mysql/foreign-key-alter/fixtures.sql | 14 + .../mysql/foreign-key-alter/specs/issues.yaml | 18 + .../tests/mysql/foreign-key-create/Dockerfile | 9 + .../tests/mysql/foreign-key-create/Makefile | 55 +++ .../expect/kustomization.yaml | 5 + .../mysql/foreign-key-create/expect/misc.yaml | 16 + .../foreign-key-create/expect/projects.yaml | 20 ++ .../expect/user-project.yaml | 47 +++ .../foreign-key-create/expect/users.yaml | 20 ++ .../mysql/foreign-key-create/fixtures.sql | 14 + .../specs/user-project.yaml | 34 ++ .../tests/mysql/foreign-key-drop/Dockerfile | 9 + .../tests/mysql/foreign-key-drop/Makefile | 55 +++ .../expect/kustomization.yaml | 3 + .../mysql/foreign-key-drop/expect/org.yaml | 20 ++ .../foreign-key-drop/expect/projects.yaml | 20 ++ .../tests/mysql/foreign-key-drop/fixtures.sql | 9 + .../mysql/foreign-key-drop/specs/org.yaml | 16 + .../tests/postgres-create/connection.yaml | 12 - .../tests/postgres-create/postgres.yaml | 120 ------- integration/tests/postgres-create/test.yaml | 26 -- .../tests/postgres-create/users-table.yaml | 20 -- .../tests/postgres/create-table/Dockerfile | 10 + .../tests/postgres/create-table/Makefile | 55 +++ .../create-table/expect/kustomization.yaml | 3 + .../postgres/create-table/expect/other.yaml | 20 ++ .../postgres/create-table/expect/users.yaml | 24 ++ .../tests/postgres/create-table/fixtures.sql | 4 + .../postgres/create-table/specs/users.yaml | 13 + .../postgres/foreign-key-alter/Dockerfile | 10 + .../tests/postgres/foreign-key-alter/Makefile | 55 +++ .../foreign-key-alter/expect/issues.yaml | 28 ++ .../expect/kustomization.yaml | 4 + .../foreign-key-alter/expect/projects.yaml | 20 ++ .../foreign-key-alter/expect/users.yaml | 20 ++ .../postgres/foreign-key-alter/fixtures.sql | 14 + .../foreign-key-alter/specs/issues.yaml | 18 + .../postgres/foreign-key-create/Dockerfile | 10 + .../postgres/foreign-key-create/Makefile | 55 +++ .../expect/kustomization.yaml | 5 + .../foreign-key-create/expect/misc.yaml | 16 + .../foreign-key-create/expect/projects.yaml | 20 ++ .../expect/user-project.yaml | 47 +++ .../foreign-key-create/expect/users.yaml | 20 ++ .../postgres/foreign-key-create/fixtures.sql | 14 + .../specs/user-project.yaml | 34 ++ .../postgres/foreign-key-drop/Dockerfile | 10 + .../tests/postgres/foreign-key-drop/Makefile | 55 +++ .../expect/kustomization.yaml | 3 + .../postgres/foreign-key-drop/expect/org.yaml | 20 ++ .../foreign-key-drop/expect/projects.yaml | 20 ++ .../postgres/foreign-key-drop/fixtures.sql | 9 + .../postgres/foreign-key-drop/specs/org.yaml | 15 + ...ysql_types.test.go => mysql_types_test.go} | 0 pkg/apis/schemas/v1alpha1/sql.go | 26 +- pkg/apis/schemas/v1alpha1/sql_types_test.go | 60 ++++ pkg/apis/schemas/v1alpha1/table_types.go | 12 +- pkg/apis/schemas/v1alpha1/table_types_test.go | 1 - .../schemas/v1alpha1/zz_generated.deepcopy.go | 54 +++ pkg/database/database.go | 12 + pkg/database/interfaces/connection.go | 12 +- pkg/database/mysql/alter.go | 15 +- pkg/database/mysql/alter_test.go | 27 +- pkg/database/mysql/column.go | 38 +- pkg/database/mysql/column_test.go | 9 +- pkg/database/mysql/create.go | 7 + pkg/database/mysql/deploy.go | 81 ++++- pkg/database/mysql/foreignkey.go | 23 ++ pkg/database/mysql/tables.go | 148 ++++++++ pkg/database/postgres/alter.go | 7 +- pkg/database/postgres/alter_test.go | 29 +- pkg/database/postgres/column.go | 38 +- pkg/database/postgres/column_test.go | 13 +- pkg/database/postgres/create.go | 11 + pkg/database/postgres/deploy.go | 87 ++++- pkg/database/postgres/foreignkey.go | 23 ++ pkg/database/postgres/tables.go | 85 ++++- pkg/database/types/column.go | 35 ++ pkg/database/types/foreignkey.go | 52 +++ pkg/generate/generate.go | 139 +++++--- pkg/generate/generate_test.go | 105 ++++++ 116 files changed, 2387 insertions(+), 1385 deletions(-) create mode 100644 config/dev/github/projects-table-pg.yaml create mode 100644 config/dev/github/projects.yaml create mode 100644 config/dev/github/users.yaml create mode 100644 integration/.gitignore delete mode 100644 integration/cmd/integration/main.go delete mode 100644 integration/manifests/manager.yaml delete mode 100644 integration/pkg/cli/root.go delete mode 100644 integration/pkg/cli/run.go delete mode 100644 integration/pkg/runner/cluster.go delete mode 100644 integration/pkg/runner/crd.go delete mode 100644 integration/pkg/runner/runner.go delete mode 100644 integration/pkg/runner/types.go delete mode 100644 integration/pkg/runner/verify.go delete mode 100644 integration/postgres/Makefile delete mode 100644 integration/postgres/README.md create mode 100644 integration/tests/mysql/create-table/Dockerfile create mode 100644 integration/tests/mysql/create-table/Makefile create mode 100644 integration/tests/mysql/create-table/expect/kustomization.yaml create mode 100644 integration/tests/mysql/create-table/expect/other.yaml create mode 100644 integration/tests/mysql/create-table/expect/users.yaml create mode 100644 integration/tests/mysql/create-table/fixtures.sql create mode 100644 integration/tests/mysql/create-table/specs/users.yaml create mode 100644 integration/tests/mysql/foreign-key-alter/Dockerfile create mode 100644 integration/tests/mysql/foreign-key-alter/Makefile create mode 100644 integration/tests/mysql/foreign-key-alter/expect/issues.yaml create mode 100644 integration/tests/mysql/foreign-key-alter/expect/kustomization.yaml create mode 100644 integration/tests/mysql/foreign-key-alter/expect/projects.yaml create mode 100644 integration/tests/mysql/foreign-key-alter/expect/users.yaml create mode 100644 integration/tests/mysql/foreign-key-alter/fixtures.sql create mode 100644 integration/tests/mysql/foreign-key-alter/specs/issues.yaml create mode 100644 integration/tests/mysql/foreign-key-create/Dockerfile create mode 100644 integration/tests/mysql/foreign-key-create/Makefile create mode 100644 integration/tests/mysql/foreign-key-create/expect/kustomization.yaml create mode 100644 integration/tests/mysql/foreign-key-create/expect/misc.yaml create mode 100644 integration/tests/mysql/foreign-key-create/expect/projects.yaml create mode 100644 integration/tests/mysql/foreign-key-create/expect/user-project.yaml create mode 100644 integration/tests/mysql/foreign-key-create/expect/users.yaml create mode 100644 integration/tests/mysql/foreign-key-create/fixtures.sql create mode 100644 integration/tests/mysql/foreign-key-create/specs/user-project.yaml create mode 100644 integration/tests/mysql/foreign-key-drop/Dockerfile create mode 100644 integration/tests/mysql/foreign-key-drop/Makefile create mode 100644 integration/tests/mysql/foreign-key-drop/expect/kustomization.yaml create mode 100644 integration/tests/mysql/foreign-key-drop/expect/org.yaml create mode 100644 integration/tests/mysql/foreign-key-drop/expect/projects.yaml create mode 100644 integration/tests/mysql/foreign-key-drop/fixtures.sql create mode 100644 integration/tests/mysql/foreign-key-drop/specs/org.yaml delete mode 100644 integration/tests/postgres-create/connection.yaml delete mode 100644 integration/tests/postgres-create/postgres.yaml delete mode 100644 integration/tests/postgres-create/test.yaml delete mode 100644 integration/tests/postgres-create/users-table.yaml create mode 100644 integration/tests/postgres/create-table/Dockerfile create mode 100644 integration/tests/postgres/create-table/Makefile create mode 100644 integration/tests/postgres/create-table/expect/kustomization.yaml create mode 100644 integration/tests/postgres/create-table/expect/other.yaml create mode 100644 integration/tests/postgres/create-table/expect/users.yaml create mode 100644 integration/tests/postgres/create-table/fixtures.sql create mode 100644 integration/tests/postgres/create-table/specs/users.yaml create mode 100644 integration/tests/postgres/foreign-key-alter/Dockerfile create mode 100644 integration/tests/postgres/foreign-key-alter/Makefile create mode 100644 integration/tests/postgres/foreign-key-alter/expect/issues.yaml create mode 100644 integration/tests/postgres/foreign-key-alter/expect/kustomization.yaml create mode 100644 integration/tests/postgres/foreign-key-alter/expect/projects.yaml create mode 100644 integration/tests/postgres/foreign-key-alter/expect/users.yaml create mode 100644 integration/tests/postgres/foreign-key-alter/fixtures.sql create mode 100644 integration/tests/postgres/foreign-key-alter/specs/issues.yaml create mode 100644 integration/tests/postgres/foreign-key-create/Dockerfile create mode 100644 integration/tests/postgres/foreign-key-create/Makefile create mode 100644 integration/tests/postgres/foreign-key-create/expect/kustomization.yaml create mode 100644 integration/tests/postgres/foreign-key-create/expect/misc.yaml create mode 100644 integration/tests/postgres/foreign-key-create/expect/projects.yaml create mode 100644 integration/tests/postgres/foreign-key-create/expect/user-project.yaml create mode 100644 integration/tests/postgres/foreign-key-create/expect/users.yaml create mode 100644 integration/tests/postgres/foreign-key-create/fixtures.sql create mode 100644 integration/tests/postgres/foreign-key-create/specs/user-project.yaml create mode 100644 integration/tests/postgres/foreign-key-drop/Dockerfile create mode 100644 integration/tests/postgres/foreign-key-drop/Makefile create mode 100644 integration/tests/postgres/foreign-key-drop/expect/kustomization.yaml create mode 100644 integration/tests/postgres/foreign-key-drop/expect/org.yaml create mode 100644 integration/tests/postgres/foreign-key-drop/expect/projects.yaml create mode 100644 integration/tests/postgres/foreign-key-drop/fixtures.sql create mode 100644 integration/tests/postgres/foreign-key-drop/specs/org.yaml rename pkg/apis/databases/v1alpha1/{mysql_types.test.go => mysql_types_test.go} (100%) create mode 100644 pkg/apis/schemas/v1alpha1/sql_types_test.go create mode 100644 pkg/database/mysql/foreignkey.go create mode 100644 pkg/database/mysql/tables.go create mode 100644 pkg/database/postgres/foreignkey.go create mode 100644 pkg/database/types/column.go create mode 100644 pkg/database/types/foreignkey.go diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 1205dcf8f..1efee8652 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -14,30 +14,14 @@ steps: - wait - - commands: - - make -C integration integration-test-image TAG="${BUILDKITE_COMMIT:0:7}" + - label: integration-tests + commands: + - make -C integration run - wait - # - commands: - # docker run -it --rm \ - # --volume /var/lib/buildkite-agent/builds/buildkite-wdxz-1/replicated/schemahero:/go/src/github.com/schemahero/schemahero \ - # --volume /var/run/docker.sock:/var/run/docker.sock \ - # --workdir /go/src/github.com/schemahero/schemahero \ - # --env BUILDKITE_JOB_ID \ - # --env BUILDKITE_BUILD_ID \ - # --env BUILDKITE_AGENT_ACCESS_TOKEN \ - # --userns=host \ - # --volume /usr/bin/buildkite-agent:/usr/bin/buildkite-agent \ - # replicated/gitops-builder:buildkite-go12-node10 \ - # /bin/sh -e -c \ - # make -C integration run - - # - wait - - commands: - bash <(curl -s https://codecov.io/bash) - branches: "master" - commands: - make snapshot-release diff --git a/README.md b/README.md index ea8834648..d06085ad4 100644 --- a/README.md +++ b/README.md @@ -6,16 +6,13 @@ [![Godoc](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)](https://godoc.org/github.com/schemahero/schemahero) [![LICENSE](https://img.shields.io/github/license/schemahero/schemahero.svg?style=flat-square)](https://github.com/schemahero/schemahero/blob/master/LICENSE) -**Note**: This is a work-in-progress that is not yet functional. SchemaHero is a an experiment right now, and does not have enough implementation to be used in any environment. Work is in progress. - ## What is SchemaHero? -SchemaHero is a Kubernetes Operator for Declarative Schema Management for various databases. SchemaHero has the following goals: +SchemaHero is a Kubernetes Operator for [Declarative Schema Management](https://schemahero.io/background/declarative-schema-management/) for [various databases](https://schemahero.io/databases/). SchemaHero has the following goals: -1. Database tables can be expressed as [Kubernetes resources](https://github.com/schemahero/schemahero/blob/master/config/samples/schemas_v1alpha1_table.yaml) that can be updated and deployed to the cluster. -2. Database migrations can be written as SQL statements, expressed as [Kubernetes resources](https://github.com/schemahero/schemahero/blob/master/config/samples/schemas_v1alpha1_migration.yaml) that can be deployed to the cluster. -3. Database schemas can be [monitored for drift](https://github.com/schemahero/schemahero/blob/master/config/samples/databases_v1alpha1_database.yaml) and brought back to the desired state automatically. -4. Schemas and migations can [require other schemas or migrations](https://github.com/schemahero/schemahero/blob/master/config/samples/schemas_v1alpha1_table.yaml#L30) instead of ordering with timestamps and/or sequences. +1. Database table schemas can be expressed as [Kubernetes resources](https://schemahero.io/how-to-use/deploying-tables/creating-tables/) that can be deployed to a cluster. +2. Database schemas can be edited and deployed to the cluster. SchemaHero will calculate the required change (`ALTER TABLE` statement) and apply it. +3. SchemaHero can manage databases that are deployed to the cluster, or external to the cluster (RDS, Google CloudSQL, etc). ## Getting Started diff --git a/config/crds/schemas_v1alpha1_table.yaml b/config/crds/schemas_v1alpha1_table.yaml index d141f6dab..4443713fe 100644 --- a/config/crds/schemas_v1alpha1_table.yaml +++ b/config/crds/schemas_v1alpha1_table.yaml @@ -59,6 +59,32 @@ spec: - type type: object type: array + foreignKeys: + items: + properties: + columns: + items: + type: string + type: array + name: + type: string + references: + properties: + columns: + items: + type: string + type: array + table: + type: string + required: + - table + - columns + type: object + required: + - columns + - references + type: object + type: array isDeleted: type: boolean primaryKey: @@ -89,6 +115,32 @@ spec: - type type: object type: array + foreignKeys: + items: + properties: + columns: + items: + type: string + type: array + name: + type: string + references: + properties: + columns: + items: + type: string + type: array + table: + type: string + required: + - table + - columns + type: object + required: + - columns + - references + type: object + type: array isDeleted: type: boolean primaryKey: @@ -102,7 +154,6 @@ spec: required: - database - name - - requires - schema type: object status: diff --git a/config/dev/database/postgres-10.7.yaml b/config/dev/database/postgres-10.7.yaml index f024203d4..49f80b7b2 100644 --- a/config/dev/database/postgres-10.7.yaml +++ b/config/dev/database/postgres-10.7.yaml @@ -17,10 +17,11 @@ spec: ports: - name: postgresql port: 5432 + nodePort: 30432 targetPort: postgresql selector: app: postgresql - type: ClusterIP + type: NodePort --- apiVersion: apps/v1beta2 kind: StatefulSet diff --git a/config/dev/github/projects-table-pg.yaml b/config/dev/github/projects-table-pg.yaml new file mode 100644 index 000000000..ddcffe9bd --- /dev/null +++ b/config/dev/github/projects-table-pg.yaml @@ -0,0 +1,23 @@ +apiVersion: schemas.schemahero.io/v1alpha1 +kind: Table +metadata: + name: projects +spec: + database: testdb + name: projects + requires: [] + schema: + postgres: + primaryKey: [id] + foreignKeys: + - columns: + - id + references: + table: users + columns: + - id + columns: + - name: id + type: integer + - name: user_id + type: integer diff --git a/config/dev/github/projects.yaml b/config/dev/github/projects.yaml new file mode 100644 index 000000000..2ed8a655b --- /dev/null +++ b/config/dev/github/projects.yaml @@ -0,0 +1,20 @@ +database: testdb +name: projects +requires: [] +schema: + postgres: + primaryKey: [id] + foreignKeys: + - columns: + - id + references: + table: users + columns: + - id + columns: + - name: id + type: integer + - name: name + type: varchar(255) + - name: user_id + type: integer diff --git a/config/dev/github/users-table-pg.yaml b/config/dev/github/users-table-pg.yaml index e04c7dd0b..44bbbded5 100644 --- a/config/dev/github/users-table-pg.yaml +++ b/config/dev/github/users-table-pg.yaml @@ -1,8 +1,6 @@ apiVersion: schemas.schemahero.io/v1alpha1 kind: Table metadata: - labels: - controller-tools.k8s.io: "1.0" name: users spec: database: testdb diff --git a/config/dev/github/users.yaml b/config/dev/github/users.yaml new file mode 100644 index 000000000..b96630aa6 --- /dev/null +++ b/config/dev/github/users.yaml @@ -0,0 +1,15 @@ +database: testdb +name: users +requires: [] +schema: + postgres: + primaryKey: [id] + columns: + - name: id + type: integer + - name: login + type: varchar(255) + - name: name + type: varchar(255) + + diff --git a/integration/.gitignore b/integration/.gitignore new file mode 100644 index 000000000..89f9ac04a --- /dev/null +++ b/integration/.gitignore @@ -0,0 +1 @@ +out/ diff --git a/integration/Makefile b/integration/Makefile index dada18fee..dd471752a 100644 --- a/integration/Makefile +++ b/integration/Makefile @@ -1,28 +1,26 @@ -export GO_BUILD=env GO111MODULE=on go build -TAG ?= $(shell uuidgen) +SHELL := /bin/bash +IMAGE := "ttl.sh/$(shell uuidgen):1h" +export IMAGE .PHONY: run -run: build - bin/schemahero-integration-tests run \ - --manager-image-name "ttl.sh/sh/manager-$(TAG):1h" \ - --schemahero-image-name "ttl.sh/sh/$(TAG):1h" +run: build postgres mysql -integration-test-image: - docker build -t ttl.sh/sh/$(TAG):1h -f ../Dockerfile.schemahero .. - docker build -t ttl.sh/sh/manager-$(TAG):1h -f ../Dockerfile.manager .. - docker push ttl.sh/sh/$(TAG):1h - docker push ttl.sh/sh/manager-$(TAG):1h +.PHONY: postgres +postgres: build + make -C tests/postgres/create-table run + make -C tests/postgres/foreign-key-create run + make -C tests/postgres/foreign-key-drop run + make -C tests/postgres/foreign-key-alter run + +.PHONY: mysql +mysql: build + make -C tests/mysql/create-table run + make -C tests/mysql/foreign-key-create run + make -C tests/mysql/foreign-key-drop run + make -C tests/mysql/foreign-key-alter run .PHONY: build -build: GO111MODULE = "on" build: - rm -rf bin/schemahero-integration-tests - $(GO_BUILD) \ - -ldflags "\ - -X ${VERSION_PACKAGE}.version=${VERSION} \ - -X ${VERSION_PACKAGE}.gitSHA=${GIT_SHA} \ - -X ${VERSION_PACKAGE}.buildTime=${DATE}" \ - -o bin/schemahero-integration-tests \ - ./cmd/integration - @echo "built bin/schemahero-integration-tests" + docker build -t $(IMAGE) -f ../Dockerfile.schemahero .. + docker push $(IMAGE) diff --git a/integration/README.md b/integration/README.md index f029adc81..15bf4663a 100644 --- a/integration/README.md +++ b/integration/README.md @@ -1,2 +1,11 @@ # Schemahero Integration Tests +These tests verify SchemaHero by: + +1. Creating a database with an init script so that there are some predefined tables +2. Applying a table.yaml +3. Generating fixtures using schemahero +4. Verifying that the generated fixtures are correct + + +Ideally these tests should be executed from Go code for more reliability and easier to use. diff --git a/integration/cmd/integration/main.go b/integration/cmd/integration/main.go deleted file mode 100644 index 81f3c2038..000000000 --- a/integration/cmd/integration/main.go +++ /dev/null @@ -1,7 +0,0 @@ -package main - -import "github.com/schemahero/schemahero/integration/pkg/cli" - -func main() { - cli.InitAndExecute() -} diff --git a/integration/manifests/manager.yaml b/integration/manifests/manager.yaml deleted file mode 100644 index b9d17cbdd..000000000 --- a/integration/manifests/manager.yaml +++ /dev/null @@ -1,327 +0,0 @@ -### GENERATED WITH kustomize build ../config/default - -apiVersion: v1 -kind: Namespace -metadata: - labels: - control-plane: controller-manager - controller-tools.k8s.io: "1.0" - name: schemahero-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - creationTimestamp: null - name: schemahero-manager-role -rules: -- apiGroups: - - apps - resources: - - deployments - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - apps - resources: - - deployments/status - verbs: - - get - - update - - patch -- apiGroups: - - databases.schemahero.io - resources: - - databases - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - databases.schemahero.io - resources: - - databases/status - verbs: - - get - - update - - patch -- apiGroups: - - apps - resources: - - deployments - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - apps - resources: - - deployments/status - verbs: - - get - - update - - patch -- apiGroups: - - schemas.schemahero.io - resources: - - migrations - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - schemas.schemahero.io - resources: - - migrations/status - verbs: - - get - - update - - patch -- apiGroups: - - apps - resources: - - deployments - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - apps - resources: - - deployments/status - verbs: - - get - - update - - patch -- apiGroups: - - schemas.schemahero.io - resources: - - tables - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - schemas.schemahero.io - resources: - - tables/status - verbs: - - get - - update - - patch -- apiGroups: - - admissionregistration.k8s.io - resources: - - mutatingwebhookconfigurations - - validatingwebhookconfigurations - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch - - create - - update - - patch - - delete ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: schemahero-proxy-role -rules: -- apiGroups: - - authentication.k8s.io - resources: - - tokenreviews - verbs: - - create -- apiGroups: - - authorization.k8s.io - resources: - - subjectaccessreviews - verbs: - - create ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - creationTimestamp: null - name: schemahero-manager-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: schemahero-manager-role -subjects: -- kind: ServiceAccount - name: default - namespace: schemahero-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: schemahero-proxy-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: schemahero-proxy-role -subjects: -- kind: ServiceAccount - name: default - namespace: schemahero-system ---- -apiVersion: v1 -kind: Secret -metadata: - name: schemahero-webhook-server-secret - namespace: schemahero-system ---- -apiVersion: v1 -kind: Service -metadata: - annotations: - prometheus.io/port: "8443" - prometheus.io/scheme: https - prometheus.io/scrape: "true" - labels: - control-plane: controller-manager - controller-tools.k8s.io: "1.0" - name: schemahero-controller-manager-metrics-service - namespace: schemahero-system -spec: - ports: - - name: https - port: 8443 - targetPort: https - selector: - control-plane: controller-manager - controller-tools.k8s.io: "1.0" ---- -apiVersion: v1 -kind: Service -metadata: - labels: - control-plane: controller-manager - controller-tools.k8s.io: "1.0" - name: schemahero-controller-manager-service - namespace: schemahero-system -spec: - ports: - - port: 443 - selector: - control-plane: controller-manager - controller-tools.k8s.io: "1.0" ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - labels: - control-plane: controller-manager - controller-tools.k8s.io: "1.0" - name: schemahero-controller-manager - namespace: schemahero-system -spec: - selector: - matchLabels: - control-plane: controller-manager - controller-tools.k8s.io: "1.0" - serviceName: schemahero-controller-manager-service - template: - metadata: - labels: - control-plane: controller-manager - controller-tools.k8s.io: "1.0" - spec: - containers: - - args: - - --secure-listen-address=0.0.0.0:8443 - - --upstream=http://127.0.0.1:8088/ - - --logtostderr=true - - --v=10 - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.4.0 - name: kube-rbac-proxy - ports: - - containerPort: 8443 - name: https - - args: - - --metrics-addr=127.0.0.1:8088 - command: - - /manager - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: SECRET_NAME - value: schemahero-webhook-server-secret - image: schemahero/schemahero-manager:latest - imagePullPolicy: Always - name: manager - ports: - - containerPort: 9876 - name: webhook-server - protocol: TCP - resources: - limits: - cpu: 100m - memory: 30Mi - requests: - cpu: 100m - memory: 20Mi - volumeMounts: - - mountPath: /tmp/cert - name: cert - readOnly: true - terminationGracePeriodSeconds: 10 - volumes: - - name: cert - secret: - defaultMode: 420 - secretName: schemahero-webhook-server-secret diff --git a/integration/pkg/cli/root.go b/integration/pkg/cli/root.go deleted file mode 100644 index 5f5603970..000000000 --- a/integration/pkg/cli/root.go +++ /dev/null @@ -1,43 +0,0 @@ -package cli - -import ( - "fmt" - "os" - "strings" - - "github.com/spf13/cobra" - "github.com/spf13/viper" -) - -func RootCmd() *cobra.Command { - cmd := &cobra.Command{ - Use: "schemahero-integration-tests", - Short: "SchemaHero Integration Tests", - Long: `...`, - SilenceUsage: true, - SilenceErrors: true, - Run: func(cmd *cobra.Command, args []string) { - cmd.Help() - os.Exit(1) - }, - } - - cobra.OnInitialize(initConfig) - - cmd.AddCommand(Run()) - - viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) - return cmd -} - -func InitAndExecute() { - if err := RootCmd().Execute(); err != nil { - fmt.Println(err) - os.Exit(1) - } -} - -func initConfig() { - viper.SetEnvPrefix("SCHEMAHERO") - viper.AutomaticEnv() -} diff --git a/integration/pkg/cli/run.go b/integration/pkg/cli/run.go deleted file mode 100644 index f072fc40a..000000000 --- a/integration/pkg/cli/run.go +++ /dev/null @@ -1,27 +0,0 @@ -package cli - -import ( - "github.com/schemahero/schemahero/integration/pkg/runner" - - "github.com/spf13/cobra" - "github.com/spf13/viper" -) - -func Run() *cobra.Command { - cmd := &cobra.Command{ - Use: "run", - Short: "run the integration tests", - Long: `...`, - RunE: func(cmd *cobra.Command, args []string) error { - r := runner.NewRunner() - return r.RunSync() - }, - } - - cmd.Flags().String("manager-image-name", "", "docker image for the manager pod") - cmd.Flags().String("schemahero-image-name", "", "docker image for the schemahero pod") - - viper.BindPFlags(cmd.Flags()) - - return cmd -} diff --git a/integration/pkg/runner/cluster.go b/integration/pkg/runner/cluster.go deleted file mode 100644 index dd0dde22e..000000000 --- a/integration/pkg/runner/cluster.go +++ /dev/null @@ -1,242 +0,0 @@ -package runner - -import ( - "bytes" - "context" - "fmt" - "io" - "io/ioutil" - "os" - "strings" - "time" - - "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" - "github.com/docker/docker/api/types/mount" - "github.com/docker/docker/client" - "github.com/docker/docker/pkg/stdcopy" - "sigs.k8s.io/kind/pkg/cluster" - "sigs.k8s.io/kind/pkg/cluster/config/encoding" - "sigs.k8s.io/kind/pkg/cluster/create" -) - -type Cluster struct { - Name string - KubeConfigPath string - KubeConfigFromDockerPath string -} - -func createCluster(name string) (*Cluster, error) { - cfg, err := encoding.Load("") - if err != nil { - return nil, err - } - - cfg.Networking.APIServerAddress = "0.0.0.0" - - if err := cfg.Validate(); err != nil { - return nil, err - } - - ctx := cluster.NewContext(name) - err = ctx.Create(cfg, - create.Retain(true), - create.WaitForReady(time.Second*90), - create.SetupKubernetes(true)) - if err != nil { - return nil, err - } - - kubeConfig, err := ioutil.ReadFile(ctx.KubeConfigPath()) - if err != nil { - return nil, err - } - - rewrittenKubeConfig := strings.Replace(string(kubeConfig), "localhost", "kubernetes", -1) - tmpFile, err := ioutil.TempFile(os.TempDir(), "kubeconfig-") - if err != nil { - return nil, err - } - if _, err = tmpFile.Write([]byte(rewrittenKubeConfig)); err != nil { - return nil, err - } - - if err = os.Chmod(tmpFile.Name(), 0644); err != nil { - return nil, err - } - - cluster := Cluster{ - Name: name, - KubeConfigPath: ctx.KubeConfigPath(), - KubeConfigFromDockerPath: tmpFile.Name(), - } - - return &cluster, nil -} - -func (c Cluster) delete() error { - os.Remove(c.KubeConfigFromDockerPath) - ctx := cluster.NewContext(c.Name) - return ctx.Delete() -} - -func (c Cluster) kubectl(ctx context.Context, cmd []string) (*container.Config, *container.HostConfig, error) { - cli, err := client.NewEnvClient() - if err != nil { - return nil, nil, err - } - - pullReader, err := cli.ImagePull(ctx, "docker.io/bitnami/kubectl:1.14", types.ImagePullOptions{}) - if err != nil { - return nil, nil, err - } - io.Copy(ioutil.Discard, pullReader) - - containerConfig := &container.Config{ - Image: "bitnami/kubectl:1.14", - Env: []string{ - "KUBECONFIG=/kubeconfig", - }, - Cmd: cmd, - } - hostConfig := &container.HostConfig{ - Mounts: []mount.Mount{ - { - Type: "bind", - Source: c.KubeConfigFromDockerPath, - Target: "/kubeconfig", - }, - }, - ExtraHosts: []string{ - "kubernetes:172.17.0.1", - }, - } - - return containerConfig, hostConfig, nil -} - -func (c Cluster) apply(manifests []byte, showStdOut bool) error { - ctx := context.Background() - - tmpFile, err := ioutil.TempFile(os.TempDir(), "manifests-") - if err != nil { - return err - } - defer os.Remove(tmpFile.Name()) - if _, err = tmpFile.Write(manifests); err != nil { - return err - } - if err = os.Chmod(tmpFile.Name(), 0644); err != nil { - return err - } - - cmd := []string{ - "apply", - "-f", - "/manifests.yaml", - } - containerConfig, hostConfig, err := c.kubectl(ctx, cmd) - if err != nil { - return err - } - - hostConfig.Mounts = append(hostConfig.Mounts, - mount.Mount{ - Type: "bind", - Source: tmpFile.Name(), - Target: "/manifests.yaml", - }) - - cli, err := client.NewEnvClient() - if err != nil { - return err - } - - resp, err := cli.ContainerCreate(ctx, containerConfig, hostConfig, nil, "") - if err != nil { - return err - } - defer cli.ContainerRemove(ctx, resp.ID, types.ContainerRemoveOptions{}) - - startOptions := types.ContainerStartOptions{} - err = cli.ContainerStart(ctx, resp.ID, startOptions) - if err != nil { - return err - } - - exitCode, err := cli.ContainerWait(ctx, resp.ID) - if err != nil { - return err - } - - data, err := cli.ContainerLogs(ctx, resp.ID, types.ContainerLogsOptions{ShowStdout: true, ShowStderr: true}) - if err != nil { - return err - } - - stdOut := new(bytes.Buffer) - stdErr := new(bytes.Buffer) - - stdcopy.StdCopy(stdOut, stdErr, data) - - if exitCode != 0 { - return fmt.Errorf("unexpected exit code running kubectl: %d\nstderr:%s\b\bstdout:%s", exitCode, stdErr, stdOut) - } - - if showStdOut { - fmt.Printf("%s\n", stdOut) - } - - return nil -} - -func (c Cluster) exec(podName string, command string, args []string) (int64, []byte, []byte, error) { - ctx := context.Background() - - cmd := []string{ - "exec", - podName, - command, - "--", - } - cmd = append(cmd, args...) - - containerConfig, hostConfig, err := c.kubectl(ctx, cmd) - if err != nil { - return -1, nil, nil, err - } - - cli, err := client.NewEnvClient() - if err != nil { - return -1, nil, nil, err - } - - resp, err := cli.ContainerCreate(ctx, containerConfig, hostConfig, nil, "") - if err != nil { - return -1, nil, nil, err - } - defer cli.ContainerRemove(ctx, resp.ID, types.ContainerRemoveOptions{}) - - startOptions := types.ContainerStartOptions{} - err = cli.ContainerStart(ctx, resp.ID, startOptions) - if err != nil { - return -1, nil, nil, err - } - - exitCode, err := cli.ContainerWait(ctx, resp.ID) - if err != nil { - return -1, nil, nil, err - } - - data, err := cli.ContainerLogs(ctx, resp.ID, types.ContainerLogsOptions{ShowStdout: true, ShowStderr: true}) - if err != nil { - return -1, nil, nil, err - } - - stdOut := new(bytes.Buffer) - stdErr := new(bytes.Buffer) - - stdcopy.StdCopy(stdOut, stdErr, data) - - return exitCode, stdOut.Bytes(), stdErr.Bytes(), nil -} diff --git a/integration/pkg/runner/crd.go b/integration/pkg/runner/crd.go deleted file mode 100644 index 18428338d..000000000 --- a/integration/pkg/runner/crd.go +++ /dev/null @@ -1,50 +0,0 @@ -package runner - -import ( - "fmt" - "io/ioutil" - "path/filepath" - "strings" -) - -func getApplyableOperator(managerImageName string) ([]byte, error) { - manager, err := getApplyableManager(managerImageName) - if err != nil { - return nil, err - } - crds, err := getApplyableCrds() - if err != nil { - return nil, err - } - - return append(manager, crds...), nil -} - -func getApplyableManager(managerImageName string) ([]byte, error) { - crd, err := ioutil.ReadFile("manifests/manager.yaml") - if err != nil { - return nil, err - } - - updatedCrd := strings.Replace(string(crd), "schemahero/schemahero-manager:latest", managerImageName, -1) - return []byte(updatedCrd), nil -} - -func getApplyableCrds() ([]byte, error) { - crds, err := ioutil.ReadDir("../config/crds") - if err != nil { - return nil, err - } - - manifests := "---" - for _, crd := range crds { - manifest, err := ioutil.ReadFile(filepath.Join("../config/crds/", crd.Name())) - if err != nil { - return nil, err - } - - manifests = fmt.Sprintf("%s\n%s\n---", manifests, manifest) - } - - return []byte(manifests), nil -} diff --git a/integration/pkg/runner/runner.go b/integration/pkg/runner/runner.go deleted file mode 100644 index 2c46ac24a..000000000 --- a/integration/pkg/runner/runner.go +++ /dev/null @@ -1,151 +0,0 @@ -package runner - -import ( - "errors" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "strings" - "time" - - "github.com/spf13/viper" -) - -type Runner struct { - Viper *viper.Viper -} - -func NewRunner() *Runner { - return &Runner{ - Viper: viper.GetViper(), - } -} - -func (r *Runner) RunSync() error { - fmt.Println("running integration tests") - - currentDir, err := os.Getwd() - if err != nil { - return err - } - tests, err := ioutil.ReadDir(filepath.Join(currentDir, "tests")) - if err != nil { - return err - } - - for _, testFile := range tests { - if testFile.IsDir() { - fmt.Printf("-----> Beginning test %q\n", testFile.Name()) - - root := filepath.Join(currentDir, "tests", testFile.Name()) - - test, err := unmarshalTestFile(filepath.Join(root, "test.yaml")) - if err != nil { - return err - } - - cluster, err := createCluster(test.Cluster.Name) - if err != nil { - return err - } - if test.Cluster.SkipCleanup { - fmt.Printf("\n\n WARNING: Skipping Cleanup of %s\n\n\n", test.Cluster.Name) - } else { - defer func() { - fmt.Printf("(%s) -----> Deleting cluster\n", test.Cluster.Name) - cluster.delete() - }() - } - - fmt.Printf("(%s) -----> Applying databases\n", test.Cluster.Name) - for _, database := range test.Databases { - fmt.Printf("(%s) -----> ... %s\n", test.Cluster.Name, database) - databaseManifests, err := ioutil.ReadFile(filepath.Join(root, database)) - if err != nil { - return err - } - - if err := cluster.apply(databaseManifests, false); err != nil { - return err - } - } - - // TODO the database has to be started before continuing here... - // The framework (really the operator itself) should handle this - time.Sleep(time.Second * 30) - - fmt.Printf("(%s) -----> Applying SchemaHero Operator\n", test.Cluster.Name) - operator, err := getApplyableOperator(r.Viper.GetString("manager-image-name")) - if err != nil { - return nil - } - if err := cluster.apply(operator, false); err != nil { - return err - } - - // Give the cluster 2 seconds to register the CRDs. This shouldn't be necessary - // And for production code this would be better handled in almost any other way - // But this test flow is a very specific pattern and this is working for now. - time.Sleep(time.Second * 2) - - fmt.Printf("(%s) -----> Applying database connections\n", test.Cluster.Name) - for _, connection := range test.Connections { - fmt.Printf("(%s) -----> ... %s\n", test.Cluster.Name, connection) - connectionManifests, err := ioutil.ReadFile(filepath.Join(root, connection)) - if err != nil { - return err - } - - replacedConnetionManifests := strings.Replace(string(connectionManifests), "__SCHEMAHERO_IMAGE_NAME__", r.Viper.GetString("schemahero-image-name"), -1) - if err := cluster.apply([]byte(replacedConnetionManifests), false); err != nil { - return err - } - } - - fmt.Printf("(%s) -----> Setting up test\n", test.Cluster.Name) - // TODO - - fmt.Printf("(%s) -----> Running test(s)\n", test.Cluster.Name) - for _, testStep := range test.Steps { - fmt.Printf("(%s) -----> ... %s\n", test.Cluster.Name, testStep.Name) - - if testStep.Table != nil { - sourceManifests, err := ioutil.ReadFile(filepath.Join(root, testStep.Table.Source)) - if err != nil { - return err - } - - if err := cluster.apply(sourceManifests, true); err != nil { - return err - } - - // Wait up to 5 seconds for verification to pass - verifyCheckCount := 0 - maxVerifications := 10 - for verifyCheckCount < maxVerifications { - ok, stdout, stderr, err := verify(cluster, testStep.Verification) - if err != nil { - return err - } - - if ok { - verifyCheckCount = maxVerifications - } else { - verifyCheckCount++ - if verifyCheckCount == maxVerifications { - fmt.Printf("stderr: %s\n", stderr) - fmt.Printf("stdout: %s\n", stdout) - return errors.New("verification failed") - } else { - time.Sleep(time.Second) - } - } - } - } - } - } - } - - return nil -} diff --git a/integration/pkg/runner/types.go b/integration/pkg/runner/types.go deleted file mode 100644 index c1eaf134d..000000000 --- a/integration/pkg/runner/types.go +++ /dev/null @@ -1,53 +0,0 @@ -package runner - -import ( - "io/ioutil" - - "gopkg.in/yaml.v2" -) - -type Test struct { - Cluster TestCluster `yaml:"cluster"` - Databases []string `yaml:"databases"` - Connections []string `yaml:"connections"` - Steps []*TestStep `yaml:"steps"` -} - -type TestCluster struct { - Name string `yaml:"name"` - SkipCleanup bool `yaml:"skipCleanup"` -} - -type TestStep struct { - Name string `yaml:"name"` - Table *TestStepTable `yaml:"table"` - Verification *TestVerification `yaml:"verification"` -} - -type TestStepTable struct { - Source string `yaml:"source"` -} - -type TestVerification struct { - Exec TestExec `yaml:"exec"` -} - -type TestExec struct { - Pod string `yaml:"pod"` - Command string `yaml:"command"` - Args []string `yaml:"args"` -} - -func unmarshalTestFile(filename string) (*Test, error) { - data, err := ioutil.ReadFile(filename) - if err != nil { - return nil, err - } - - test := Test{} - if err := yaml.Unmarshal(data, &test); err != nil { - return nil, err - } - - return &test, nil -} diff --git a/integration/pkg/runner/verify.go b/integration/pkg/runner/verify.go deleted file mode 100644 index 7930e78b0..000000000 --- a/integration/pkg/runner/verify.go +++ /dev/null @@ -1,12 +0,0 @@ -package runner - -// verify should only return an error if there's an error that should stop execution -// for transient errors, return false, nil and expect a retry/backoff interval -func verify(cluster *Cluster, verification *TestVerification) (bool, []byte, []byte, error) { - exitCode, stdout, stderr, err := cluster.exec(verification.Exec.Pod, verification.Exec.Command, verification.Exec.Args) - if err != nil { - return false, nil, nil, err - } - - return exitCode == 0, stdout, stderr, nil -} diff --git a/integration/postgres/Makefile b/integration/postgres/Makefile deleted file mode 100644 index 091e36424..000000000 --- a/integration/postgres/Makefile +++ /dev/null @@ -1,25 +0,0 @@ -SHELL := /bin/bash -CLUSTER_NAME := schemahero-integration-postgres -MANAGER_IMAGE_NAME := schemahero-integration/manager:postgres - -.PHONY: test -test: cleanup-before cluster crd cleanup-after - -.PHONY: cluster -cluster: - kind create cluster --wait 90s --name $(CLUSTER_NAME) - KUBECONFIG=$$(kind get kubeconfig-path --name="$(CLUSTER_NAME)") kubectl get nodes - -.PHONY: cleanup-% -cleanup-%: - -kind delete cluster --name $(CLUSTER_NAME) > /dev/null 2>&1 ||: - -.PHONY: crd -crd: - @echo "building manager image" - docker build -t $(MANAGER_IMAGE_NAME) -f ../../Dockerfile ../../ - - @echo "updating kustomize image patch to use newly built image" - sed -i'' -e 's@image: .*@image: '"${MANAGER_IMAGE_NAME}"'@' ../../config/integration/manager_image_patch.yaml - - KUBECONFIG=$$(kind get kubeconfig-path --name="$(CLUSTER_NAME)") kubectl apply -f ../../config/crds diff --git a/integration/postgres/README.md b/integration/postgres/README.md deleted file mode 100644 index 54d9feb0c..000000000 --- a/integration/postgres/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# Postgres Integration Tests - -This directory contains custom resources and validation to ensure that a postgres database can be initialized and created, tables and schema migrations can be applied using the SchemaHero operator. - -To execute these tests, install [kind](https://github.com/kubernetes-sigs/kind) and then execute: - -``` -make test -``` - -Note, these tests are executed as part of the continuous integration process that runs from the top level Makefile. diff --git a/integration/tests/mysql/create-table/Dockerfile b/integration/tests/mysql/create-table/Dockerfile new file mode 100644 index 000000000..367f3cf19 --- /dev/null +++ b/integration/tests/mysql/create-table/Dockerfile @@ -0,0 +1,9 @@ +FROM mysql:8.0 + +ENV MYSQL_USER=schemahero +ENV MYSQL_PASSWORD=password +ENV MYSQL_DATABASE=schemahero +ENV MYSQL_RANDOM_ROOT_PASSWORD=1 + +## Insert fixtures +COPY ./fixtures.sql /docker-entrypoint-initdb.d/ diff --git a/integration/tests/mysql/create-table/Makefile b/integration/tests/mysql/create-table/Makefile new file mode 100644 index 000000000..c67e7051d --- /dev/null +++ b/integration/tests/mysql/create-table/Makefile @@ -0,0 +1,55 @@ +SHELL := /bin/bash +TEST_NAME := mysql-create-table +DATABASE_IMAGE_NAME := schemahero/database +DATABASE_CONTAINER_NAME := database + +.PHONY: run +run: + @rm -rf ./out + @mkdir ./out + @-docker rm -f $(DATABASE_CONTAINER_NAME) > /dev/null 2>&1 ||: + @-docker rm -f $(TEST_NAME) > /dev/null 2>&1 ||: + @-docker network rm $(TEST_NAME) > /dev/null 2>&1 ||: + docker network create $(TEST_NAME) + + # Fixtures + docker pull mysql:8.0 + docker build -t $(DATABASE_IMAGE_NAME) . + @-docker rm -f $(DATABASE_CONTAINER_NAME) > /dev/null 2>&1 ||: + docker run --network $(TEST_NAME) --rm -d --name $(DATABASE_CONTAINER_NAME) $(DATABASE_IMAGE_NAME) + while ! docker exec -it $(DATABASE_CONTAINER_NAME) mysqladmin ping -hlocalhost --silent; do sleep 15; done + + # Test + docker tag $(IMAGE) schemahero/schemahero:test + docker run -v `pwd`/specs:/specs \ + --network $(TEST_NAME) \ + --name $(TEST_NAME) \ + --rm \ + schemahero/schemahero:test \ + apply \ + --driver mysql \ + --uri "schemahero:password@tcp($(DATABASE_CONTAINER_NAME):3306)/schemahero?tls=false" \ + --spec-file /specs/users.yaml + + # Verify + docker run \ + --rm \ + --network $(TEST_NAME) \ + -v `pwd`/out:/out \ + -e uid=$${UID} \ + schemahero/schemahero:test \ + generate \ + --dbname schemahero \ + --namespace default \ + --driver mysql \ + --output-dir /out \ + --uri "schemahero:password@tcp($(DATABASE_CONTAINER_NAME):3306)/schemahero?tls=false" + @echo Verifying results for $(TEST_NAME) + diff --color expect out + + # Cleanup + @-sleep 5 + rm -rf ./out + @-docker rm -f $(DATABASE_CONTAINER_NAME) + @-docker network rm $(TEST_NAME) + diff --git a/integration/tests/mysql/create-table/expect/kustomization.yaml b/integration/tests/mysql/create-table/expect/kustomization.yaml new file mode 100644 index 000000000..93c9d3e20 --- /dev/null +++ b/integration/tests/mysql/create-table/expect/kustomization.yaml @@ -0,0 +1,3 @@ +resources: +- ./other.yaml +- ./users.yaml diff --git a/integration/tests/mysql/create-table/expect/other.yaml b/integration/tests/mysql/create-table/expect/other.yaml new file mode 100644 index 000000000..b4291e623 --- /dev/null +++ b/integration/tests/mysql/create-table/expect/other.yaml @@ -0,0 +1,20 @@ +apiVersion: schemas.schemahero.io/v1alpha1 +kind: Table +metadata: + name: other +spec: + database: schemahero + name: other + schema: + mysql: + primaryKey: + - id + columns: + - name: id + type: int + constraints: + notNull: true + - name: something + type: varchar (255) + constraints: + notNull: true diff --git a/integration/tests/mysql/create-table/expect/users.yaml b/integration/tests/mysql/create-table/expect/users.yaml new file mode 100644 index 000000000..6f1d9cbdd --- /dev/null +++ b/integration/tests/mysql/create-table/expect/users.yaml @@ -0,0 +1,24 @@ +apiVersion: schemas.schemahero.io/v1alpha1 +kind: Table +metadata: + name: users +spec: + database: schemahero + name: users + schema: + mysql: + primaryKey: + - id + columns: + - name: id + type: int + constraints: + notNull: true + - name: login + type: varchar (255) + constraints: + notNull: false + - name: name + type: varchar (255) + constraints: + notNull: false diff --git a/integration/tests/mysql/create-table/fixtures.sql b/integration/tests/mysql/create-table/fixtures.sql new file mode 100644 index 000000000..19b3b4010 --- /dev/null +++ b/integration/tests/mysql/create-table/fixtures.sql @@ -0,0 +1,4 @@ +create table other ( + id integer primary key not null, + something varchar(255) not null +); diff --git a/integration/tests/mysql/create-table/specs/users.yaml b/integration/tests/mysql/create-table/specs/users.yaml new file mode 100644 index 000000000..9469fbb38 --- /dev/null +++ b/integration/tests/mysql/create-table/specs/users.yaml @@ -0,0 +1,13 @@ +database: schemahero +name: users +requires: [] +schema: + mysql: + primaryKey: [id] + columns: + - name: id + type: integer + - name: login + type: varchar(255) + - name: name + type: varchar(255) diff --git a/integration/tests/mysql/foreign-key-alter/Dockerfile b/integration/tests/mysql/foreign-key-alter/Dockerfile new file mode 100644 index 000000000..367f3cf19 --- /dev/null +++ b/integration/tests/mysql/foreign-key-alter/Dockerfile @@ -0,0 +1,9 @@ +FROM mysql:8.0 + +ENV MYSQL_USER=schemahero +ENV MYSQL_PASSWORD=password +ENV MYSQL_DATABASE=schemahero +ENV MYSQL_RANDOM_ROOT_PASSWORD=1 + +## Insert fixtures +COPY ./fixtures.sql /docker-entrypoint-initdb.d/ diff --git a/integration/tests/mysql/foreign-key-alter/Makefile b/integration/tests/mysql/foreign-key-alter/Makefile new file mode 100644 index 000000000..8a6ea923c --- /dev/null +++ b/integration/tests/mysql/foreign-key-alter/Makefile @@ -0,0 +1,54 @@ +SHELL := /bin/bash +TEST_NAME := mysql-foreign-key +DATABASE_IMAGE_NAME := schemahero/database +DATABASE_CONTAINER_NAME := database + +.PHONY: run +run: + @rm -rf ./out + @mkdir ./out + @-docker rm -f $(DATABASE_CONTAINER_NAME) > /dev/null 2>&1 ||: + @-docker rm -f $(TEST_NAME) > /dev/null 2>&1 ||: + @-docker network rm $(TEST_NAME) > /dev/null 2>&1 ||: + docker network create $(TEST_NAME) + + # Fixtures + docker pull mysql:8.0 + docker build -t $(DATABASE_IMAGE_NAME) . + @-docker rm -f $(DATABASE_CONTAINER_NAME) > /dev/null 2>&1 ||: + docker run --network $(TEST_NAME) --rm -d --name $(DATABASE_CONTAINER_NAME) $(DATABASE_IMAGE_NAME) + while ! docker exec -it $(DATABASE_CONTAINER_NAME) mysqladmin ping -hlocalhost --silent; do sleep 15; done + + # Test + docker tag $(IMAGE) schemahero/schemahero:test + docker run -v `pwd`/specs:/specs \ + --network $(TEST_NAME) \ + --name $(TEST_NAME) \ + --rm \ + schemahero/schemahero:test \ + apply \ + --driver mysql \ + --uri "schemahero:password@tcp($(DATABASE_CONTAINER_NAME):3306)/schemahero?tls=false" \ + --spec-file /specs/issues.yaml + # Verify + docker run \ + --rm \ + --network $(TEST_NAME) \ + -v `pwd`/out:/out \ + -e uid=$${UID} \ + schemahero/schemahero:test \ + generate \ + --dbname schemahero \ + --namespace default \ + --driver mysql \ + --output-dir /out \ + --uri "schemahero:password@tcp($(DATABASE_CONTAINER_NAME):3306)/schemahero?tls=false" + @echo Verifying results for $(TEST_NAME) + diff --color expect out + + # Cleanup + @-sleep 5 + rm -rf ./out + @-docker rm -f $(DATABASE_CONTAINER_NAME) + @-docker network rm $(TEST_NAME) + diff --git a/integration/tests/mysql/foreign-key-alter/expect/issues.yaml b/integration/tests/mysql/foreign-key-alter/expect/issues.yaml new file mode 100644 index 000000000..b4c59aa16 --- /dev/null +++ b/integration/tests/mysql/foreign-key-alter/expect/issues.yaml @@ -0,0 +1,28 @@ +apiVersion: schemas.schemahero.io/v1alpha1 +kind: Table +metadata: + name: issues +spec: + database: schemahero + name: issues + schema: + mysql: + primaryKey: + - id + foreignKeys: + - columns: + - project_id + references: + table: projects + columns: + - id + name: renamed_fkey + columns: + - name: id + type: int + constraints: + notNull: true + - name: project_id + type: int + constraints: + notNull: false diff --git a/integration/tests/mysql/foreign-key-alter/expect/kustomization.yaml b/integration/tests/mysql/foreign-key-alter/expect/kustomization.yaml new file mode 100644 index 000000000..0a23a2248 --- /dev/null +++ b/integration/tests/mysql/foreign-key-alter/expect/kustomization.yaml @@ -0,0 +1,4 @@ +resources: +- ./issues.yaml +- ./projects.yaml +- ./users.yaml diff --git a/integration/tests/mysql/foreign-key-alter/expect/projects.yaml b/integration/tests/mysql/foreign-key-alter/expect/projects.yaml new file mode 100644 index 000000000..d92a993ef --- /dev/null +++ b/integration/tests/mysql/foreign-key-alter/expect/projects.yaml @@ -0,0 +1,20 @@ +apiVersion: schemas.schemahero.io/v1alpha1 +kind: Table +metadata: + name: projects +spec: + database: schemahero + name: projects + schema: + mysql: + primaryKey: + - id + columns: + - name: id + type: int + constraints: + notNull: true + - name: name + type: varchar (255) + constraints: + notNull: true diff --git a/integration/tests/mysql/foreign-key-alter/expect/users.yaml b/integration/tests/mysql/foreign-key-alter/expect/users.yaml new file mode 100644 index 000000000..d00c2f8e6 --- /dev/null +++ b/integration/tests/mysql/foreign-key-alter/expect/users.yaml @@ -0,0 +1,20 @@ +apiVersion: schemas.schemahero.io/v1alpha1 +kind: Table +metadata: + name: users +spec: + database: schemahero + name: users + schema: + mysql: + primaryKey: + - id + columns: + - name: id + type: int + constraints: + notNull: true + - name: email + type: varchar (255) + constraints: + notNull: true diff --git a/integration/tests/mysql/foreign-key-alter/fixtures.sql b/integration/tests/mysql/foreign-key-alter/fixtures.sql new file mode 100644 index 000000000..f9f808d91 --- /dev/null +++ b/integration/tests/mysql/foreign-key-alter/fixtures.sql @@ -0,0 +1,14 @@ +create table users ( + id integer primary key not null, + email varchar(255) not null +); + +create table projects ( + id integer primary key not null, + name varchar(255) not null +); + +create table issues ( + id integer primary key not null, + project_id integer references users(id) +); diff --git a/integration/tests/mysql/foreign-key-alter/specs/issues.yaml b/integration/tests/mysql/foreign-key-alter/specs/issues.yaml new file mode 100644 index 000000000..d98a32231 --- /dev/null +++ b/integration/tests/mysql/foreign-key-alter/specs/issues.yaml @@ -0,0 +1,18 @@ +database: schemahero +name: issues +schema: + mysql: + primaryKey: [id] + foreignKeys: + - columns: + - project_id + references: + table: projects + columns: + - id + name: renamed_fkey + columns: + - name: id + type: integer + - name: project_id + type: integer diff --git a/integration/tests/mysql/foreign-key-create/Dockerfile b/integration/tests/mysql/foreign-key-create/Dockerfile new file mode 100644 index 000000000..367f3cf19 --- /dev/null +++ b/integration/tests/mysql/foreign-key-create/Dockerfile @@ -0,0 +1,9 @@ +FROM mysql:8.0 + +ENV MYSQL_USER=schemahero +ENV MYSQL_PASSWORD=password +ENV MYSQL_DATABASE=schemahero +ENV MYSQL_RANDOM_ROOT_PASSWORD=1 + +## Insert fixtures +COPY ./fixtures.sql /docker-entrypoint-initdb.d/ diff --git a/integration/tests/mysql/foreign-key-create/Makefile b/integration/tests/mysql/foreign-key-create/Makefile new file mode 100644 index 000000000..0c1193ecb --- /dev/null +++ b/integration/tests/mysql/foreign-key-create/Makefile @@ -0,0 +1,55 @@ +SHELL := /bin/bash +TEST_NAME := mysql-foreign-key +DATABASE_IMAGE_NAME := schemahero/database +DATABASE_CONTAINER_NAME := database + +.PHONY: run +run: + @rm -rf ./out + @mkdir ./out + @-docker rm -f $(DATABASE_CONTAINER_NAME) > /dev/null 2>&1 ||: + @-docker rm -f $(TEST_NAME) > /dev/null 2>&1 ||: + @-docker network rm $(TEST_NAME) > /dev/null 2>&1 ||: + docker network create $(TEST_NAME) + + # Fixtures + docker pull mysql:8.0 + docker build -t $(DATABASE_IMAGE_NAME) . + @-docker rm -f $(DATABASE_CONTAINER_NAME) > /dev/null 2>&1 ||: + docker run --network $(TEST_NAME) --rm -d --name $(DATABASE_CONTAINER_NAME) $(DATABASE_IMAGE_NAME) + while ! docker exec -it $(DATABASE_CONTAINER_NAME) mysqladmin ping -hlocalhost --silent; do sleep 15; done + + # Test + docker tag $(IMAGE) schemahero/schemahero:test + docker run -v `pwd`/specs:/specs \ + --network $(TEST_NAME) \ + --name $(TEST_NAME) \ + --rm \ + schemahero/schemahero:test \ + apply \ + --driver mysql \ + --uri "schemahero:password@tcp($(DATABASE_CONTAINER_NAME):3306)/schemahero?tls=false" \ + --spec-file /specs/user-project.yaml + + # Verify + docker run \ + --rm \ + --network $(TEST_NAME) \ + -v `pwd`/out:/out \ + -e uid=$${UID} \ + schemahero/schemahero:test \ + generate \ + --dbname schemahero \ + --namespace default \ + --driver mysql \ + --output-dir /out \ + --uri "schemahero:password@tcp($(DATABASE_CONTAINER_NAME):3306)/schemahero?tls=false" + @echo Verifying results for $(TEST_NAME) + diff --color expect out + + # Cleanup + @-sleep 5 + rm -rf ./out + @-docker rm -f $(DATABASE_CONTAINER_NAME) + @-docker network rm $(TEST_NAME) + diff --git a/integration/tests/mysql/foreign-key-create/expect/kustomization.yaml b/integration/tests/mysql/foreign-key-create/expect/kustomization.yaml new file mode 100644 index 000000000..c497fa5f0 --- /dev/null +++ b/integration/tests/mysql/foreign-key-create/expect/kustomization.yaml @@ -0,0 +1,5 @@ +resources: +- ./misc.yaml +- ./projects.yaml +- ./user-project.yaml +- ./users.yaml diff --git a/integration/tests/mysql/foreign-key-create/expect/misc.yaml b/integration/tests/mysql/foreign-key-create/expect/misc.yaml new file mode 100644 index 000000000..37a36554b --- /dev/null +++ b/integration/tests/mysql/foreign-key-create/expect/misc.yaml @@ -0,0 +1,16 @@ +apiVersion: schemas.schemahero.io/v1alpha1 +kind: Table +metadata: + name: misc +spec: + database: schemahero + name: misc + schema: + mysql: + primaryKey: + - pk + columns: + - name: pk + type: varchar (255) + constraints: + notNull: true diff --git a/integration/tests/mysql/foreign-key-create/expect/projects.yaml b/integration/tests/mysql/foreign-key-create/expect/projects.yaml new file mode 100644 index 000000000..d92a993ef --- /dev/null +++ b/integration/tests/mysql/foreign-key-create/expect/projects.yaml @@ -0,0 +1,20 @@ +apiVersion: schemas.schemahero.io/v1alpha1 +kind: Table +metadata: + name: projects +spec: + database: schemahero + name: projects + schema: + mysql: + primaryKey: + - id + columns: + - name: id + type: int + constraints: + notNull: true + - name: name + type: varchar (255) + constraints: + notNull: true diff --git a/integration/tests/mysql/foreign-key-create/expect/user-project.yaml b/integration/tests/mysql/foreign-key-create/expect/user-project.yaml new file mode 100644 index 000000000..9ab95af69 --- /dev/null +++ b/integration/tests/mysql/foreign-key-create/expect/user-project.yaml @@ -0,0 +1,47 @@ +apiVersion: schemas.schemahero.io/v1alpha1 +kind: Table +metadata: + name: user-project +spec: + database: schemahero + name: user_project + schema: + mysql: + primaryKey: + - project_id + - user_id + foreignKeys: + - columns: + - user_id + references: + table: users + columns: + - id + name: user_project_ibfk_1 + - columns: + - project_id + references: + table: projects + columns: + - id + name: user_project_ibfk_2 + - columns: + - misc_id + references: + table: misc + columns: + - pk + name: user_project_ibfk_3 + columns: + - name: user_id + type: int + constraints: + notNull: true + - name: project_id + type: int + constraints: + notNull: true + - name: misc_id + type: varchar (255) + constraints: + notNull: true diff --git a/integration/tests/mysql/foreign-key-create/expect/users.yaml b/integration/tests/mysql/foreign-key-create/expect/users.yaml new file mode 100644 index 000000000..d00c2f8e6 --- /dev/null +++ b/integration/tests/mysql/foreign-key-create/expect/users.yaml @@ -0,0 +1,20 @@ +apiVersion: schemas.schemahero.io/v1alpha1 +kind: Table +metadata: + name: users +spec: + database: schemahero + name: users + schema: + mysql: + primaryKey: + - id + columns: + - name: id + type: int + constraints: + notNull: true + - name: email + type: varchar (255) + constraints: + notNull: true diff --git a/integration/tests/mysql/foreign-key-create/fixtures.sql b/integration/tests/mysql/foreign-key-create/fixtures.sql new file mode 100644 index 000000000..1d73f1726 --- /dev/null +++ b/integration/tests/mysql/foreign-key-create/fixtures.sql @@ -0,0 +1,14 @@ +create table users ( + id integer primary key not null, + email varchar(255) not null +); + +create table projects ( + id integer primary key not null, + name varchar(255) not null +); + +create table misc ( + pk varchar(255) primary key not null +); + diff --git a/integration/tests/mysql/foreign-key-create/specs/user-project.yaml b/integration/tests/mysql/foreign-key-create/specs/user-project.yaml new file mode 100644 index 000000000..ee3bf842d --- /dev/null +++ b/integration/tests/mysql/foreign-key-create/specs/user-project.yaml @@ -0,0 +1,34 @@ +database: schemahero +name: user_project +schema: + mysql: + primaryKey: [user_id, project_id] + foreignKeys: + - columns: + - user_id + references: + table: users + columns: + - id + - columns: + - project_id + references: + table: projects + columns: + - id + - columns: + - misc_id + references: + table: misc + columns: + - pk + name: misc_named_fk + columns: + - name: user_id + type: integer + - name: project_id + type: integer + - name: misc_id + type: varchar(255) + constraints: + notNull: true diff --git a/integration/tests/mysql/foreign-key-drop/Dockerfile b/integration/tests/mysql/foreign-key-drop/Dockerfile new file mode 100644 index 000000000..367f3cf19 --- /dev/null +++ b/integration/tests/mysql/foreign-key-drop/Dockerfile @@ -0,0 +1,9 @@ +FROM mysql:8.0 + +ENV MYSQL_USER=schemahero +ENV MYSQL_PASSWORD=password +ENV MYSQL_DATABASE=schemahero +ENV MYSQL_RANDOM_ROOT_PASSWORD=1 + +## Insert fixtures +COPY ./fixtures.sql /docker-entrypoint-initdb.d/ diff --git a/integration/tests/mysql/foreign-key-drop/Makefile b/integration/tests/mysql/foreign-key-drop/Makefile new file mode 100644 index 000000000..dfd40dd80 --- /dev/null +++ b/integration/tests/mysql/foreign-key-drop/Makefile @@ -0,0 +1,55 @@ +SHELL := /bin/bash +TEST_NAME := mysql-foreign-key +DATABASE_IMAGE_NAME := schemahero/database +DATABASE_CONTAINER_NAME := database + +.PHONY: run +run: + @rm -rf ./out + @mkdir ./out + @-docker rm -f $(DATABASE_CONTAINER_NAME) > /dev/null 2>&1 ||: + @-docker rm -f $(TEST_NAME) > /dev/null 2>&1 ||: + @-docker network rm $(TEST_NAME) > /dev/null 2>&1 ||: + docker network create $(TEST_NAME) + + # Fixtures + docker pull mysql:8.0 + docker build -t $(DATABASE_IMAGE_NAME) . + @-docker rm -f $(DATABASE_CONTAINER_NAME) > /dev/null 2>&1 ||: + docker run --network $(TEST_NAME) --rm -d --name $(DATABASE_CONTAINER_NAME) $(DATABASE_IMAGE_NAME) + while ! docker exec -it $(DATABASE_CONTAINER_NAME) mysqladmin ping -hlocalhost --silent; do sleep 15; done + + # Test + docker tag $(IMAGE) schemahero/schemahero:test + docker run -v `pwd`/specs:/specs \ + --network $(TEST_NAME) \ + --name $(TEST_NAME) \ + --rm \ + schemahero/schemahero:test \ + apply \ + --driver mysql \ + --uri "schemahero:password@tcp($(DATABASE_CONTAINER_NAME):3306)/schemahero?tls=false" \ + --spec-file /specs/org.yaml + + # Verify + docker run \ + --rm \ + --network $(TEST_NAME) \ + -v `pwd`/out:/out \ + -e uid=$${UID} \ + schemahero/schemahero:test \ + generate \ + --dbname schemahero \ + --namespace default \ + --driver mysql \ + --output-dir /out \ + --uri "schemahero:password@tcp($(DATABASE_CONTAINER_NAME):3306)/schemahero?tls=false" + @echo Verifying results for $(TEST_NAME) + diff --color expect out + + # Cleanup + @-sleep 5 + rm -rf ./out + @-docker rm -f $(DATABASE_CONTAINER_NAME) + @-docker network rm $(TEST_NAME) + diff --git a/integration/tests/mysql/foreign-key-drop/expect/kustomization.yaml b/integration/tests/mysql/foreign-key-drop/expect/kustomization.yaml new file mode 100644 index 000000000..868cf58ab --- /dev/null +++ b/integration/tests/mysql/foreign-key-drop/expect/kustomization.yaml @@ -0,0 +1,3 @@ +resources: +- ./org.yaml +- ./projects.yaml diff --git a/integration/tests/mysql/foreign-key-drop/expect/org.yaml b/integration/tests/mysql/foreign-key-drop/expect/org.yaml new file mode 100644 index 000000000..2a1a2b173 --- /dev/null +++ b/integration/tests/mysql/foreign-key-drop/expect/org.yaml @@ -0,0 +1,20 @@ +apiVersion: schemas.schemahero.io/v1alpha1 +kind: Table +metadata: + name: org +spec: + database: schemahero + name: org + schema: + mysql: + primaryKey: + - id + columns: + - name: id + type: int + constraints: + notNull: true + - name: project_id + type: int + constraints: + notNull: true diff --git a/integration/tests/mysql/foreign-key-drop/expect/projects.yaml b/integration/tests/mysql/foreign-key-drop/expect/projects.yaml new file mode 100644 index 000000000..d92a993ef --- /dev/null +++ b/integration/tests/mysql/foreign-key-drop/expect/projects.yaml @@ -0,0 +1,20 @@ +apiVersion: schemas.schemahero.io/v1alpha1 +kind: Table +metadata: + name: projects +spec: + database: schemahero + name: projects + schema: + mysql: + primaryKey: + - id + columns: + - name: id + type: int + constraints: + notNull: true + - name: name + type: varchar (255) + constraints: + notNull: true diff --git a/integration/tests/mysql/foreign-key-drop/fixtures.sql b/integration/tests/mysql/foreign-key-drop/fixtures.sql new file mode 100644 index 000000000..aa461b4a2 --- /dev/null +++ b/integration/tests/mysql/foreign-key-drop/fixtures.sql @@ -0,0 +1,9 @@ +create table projects ( + id integer primary key not null, + name varchar(255) not null +); + +create table org ( + id integer primary key not null, + project_id integer references projects(id) +); diff --git a/integration/tests/mysql/foreign-key-drop/specs/org.yaml b/integration/tests/mysql/foreign-key-drop/specs/org.yaml new file mode 100644 index 000000000..628c419a1 --- /dev/null +++ b/integration/tests/mysql/foreign-key-drop/specs/org.yaml @@ -0,0 +1,16 @@ +database: schemahero +name: org +schema: + mysql: + primaryKey: + - id + foreignKeys: + columns: + - name: id + type: integer + constraints: + notNull: true + - name: project_id + type: integer + constraints: + notNull: true diff --git a/integration/tests/postgres-create/connection.yaml b/integration/tests/postgres-create/connection.yaml deleted file mode 100644 index 3e345e1f8..000000000 --- a/integration/tests/postgres-create/connection.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: databases.schemahero.io/v1alpha1 -kind: Database -metadata: - labels: - controller-tools.k8s.io: "1.0" - name: integrationone -schemahero: - image: __SCHEMAHERO_IMAGE_NAME__ -connection: - postgres: - uri: - value: postgres://schemahero:password@postgresql.default.svc.cluster.local:5432/integrationone?sslmode=disable diff --git a/integration/tests/postgres-create/postgres.yaml b/integration/tests/postgres-create/postgres.yaml deleted file mode 100644 index e205904a8..000000000 --- a/integration/tests/postgres-create/postgres.yaml +++ /dev/null @@ -1,120 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - labels: - app: postgresql - name: postgresql -spec: - ports: - - name: postgresql - port: 5432 - targetPort: postgresql - selector: - app: postgresql - type: ClusterIP ---- -apiVersion: apps/v1beta2 -kind: StatefulSet -metadata: - labels: - app: postgresql - name: postgresql -spec: - replicas: 1 - selector: - matchLabels: - app: postgresql - serviceName: postgresql - template: - metadata: - labels: - app: postgresql - chart: postgresql-3.18.1 - name: postgresql - spec: - containers: - - env: - - name: PGDATA - value: /bitnami/postgresql - - name: POSTGRES_USER - value: schemahero - - name: POSTGRES_PASSWORD - value: password - - name: PGPASSWORD ## This makes psql work in kubectl exec - value: password - - name: POSTGRES_DB - value: integrationone - image: docker.io/bitnami/postgresql:10.7.0 - imagePullPolicy: Always - livenessProbe: - exec: - command: - - sh - - -c - - exec pg_isready -U "schemahero" -d "integrationone" -h 127.0.0.1 - failureThreshold: 6 - initialDelaySeconds: 30 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 5 - name: postgresql - ports: - - containerPort: 5432 - name: postgresql - readinessProbe: - exec: - command: - - sh - - -c - - exec pg_isready -U "schemahero" -d "integrationone" -h 127.0.0.1 - failureThreshold: 6 - initialDelaySeconds: 5 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 5 - resources: - requests: - cpu: 500m - memory: 1024Mi - securityContext: - runAsUser: 1001 - volumeMounts: - - mountPath: /bitnami/postgresql - name: schemahero-integrationone - subPath: null - initContainers: - - command: - - sh - - -c - - | - chown -R 1001:1001 /bitnami - if [ -d /bitnami/postgresql/data ]; then - chmod 0700 /bitnami/postgresql/data; - fi - image: docker.io/bitnami/minideb:latest - imagePullPolicy: Always - name: init-chmod-data - resources: - requests: - cpu: 500m - memory: 1024Mi - securityContext: - runAsUser: 0 - volumeMounts: - - mountPath: /bitnami/postgresql - name: schemahero-integrationone - subPath: null - securityContext: - fsGroup: 1001 - volumes: [] - updateStrategy: - type: RollingUpdate - volumeClaimTemplates: - - metadata: - name: schemahero-integrationone - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi diff --git a/integration/tests/postgres-create/test.yaml b/integration/tests/postgres-create/test.yaml deleted file mode 100644 index a62cc51af..000000000 --- a/integration/tests/postgres-create/test.yaml +++ /dev/null @@ -1,26 +0,0 @@ -cluster: - name: postgres-create - # skipCleanup: true - -databases: - - postgres.yaml - -connections: - - connection.yaml - -setup: [] - -steps: - - name: create users table - table: - source: users-table.yaml - verification: - exec: - pod: postgresql-0 - command: psql - args: - - -Uschemahero - - -d - - integrationone - - -c - - 'select id, login, name from users' diff --git a/integration/tests/postgres-create/users-table.yaml b/integration/tests/postgres-create/users-table.yaml deleted file mode 100644 index 29ca65883..000000000 --- a/integration/tests/postgres-create/users-table.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: schemas.schemahero.io/v1alpha1 -kind: Table -metadata: - labels: - controller-tools.k8s.io: "1.0" - name: users -spec: - database: integrationone - name: users - requires: [] - schema: - postgres: - primaryKey: [id] - columns: - - name: id - type: integer - - name: login - type: varchar(255) - - name: name - type: varchar(255) diff --git a/integration/tests/postgres/create-table/Dockerfile b/integration/tests/postgres/create-table/Dockerfile new file mode 100644 index 000000000..a0cc4ed89 --- /dev/null +++ b/integration/tests/postgres/create-table/Dockerfile @@ -0,0 +1,10 @@ +FROM postgres:10.7 + +ENV POSTGRES_USER=schemahero +ENV POSTGRES_PASSWORD=password +ENV POSTGRES_DB=schemahero + +## Insert fixtures +COPY ./fixtures.sql /docker-entrypoint-initdb.d/ + + diff --git a/integration/tests/postgres/create-table/Makefile b/integration/tests/postgres/create-table/Makefile new file mode 100644 index 000000000..ef74474b7 --- /dev/null +++ b/integration/tests/postgres/create-table/Makefile @@ -0,0 +1,55 @@ +SHELL := /bin/bash +TEST_NAME := postgres-create-table +DATABASE_IMAGE_NAME := schemahero/database +DATABASE_CONTAINER_NAME := schemahero-database + +.PHONY: run +run: + @rm -rf ./out + @mkdir ./out + @-docker rm -f $(DATABASE_CONTAINER_NAME) > /dev/null 2>&1 ||: + @-docker rm -f $(TEST_NAME) > /dev/null 2>&1 ||: + @-docker network rm $(TEST_NAME) > /dev/null 2>&1 ||: + docker network create $(TEST_NAME) + + # Fixtures + docker pull postgres:10.7 + docker build -t $(DATABASE_IMAGE_NAME) . + @-docker rm -f $(DATABASE_CONTAINER_NAME) > /dev/null 2>&1 ||: + docker run --network $(TEST_NAME) --rm -d --name $(DATABASE_CONTAINER_NAME) $(DATABASE_IMAGE_NAME) + @-sleep 5 + + # Test + docker tag $(IMAGE) schemahero/schemahero:test + docker run -v `pwd`/specs:/specs \ + --network $(TEST_NAME) \ + --name $(TEST_NAME) \ + --rm \ + schemahero/schemahero:test \ + apply \ + --driver postgres \ + --uri postgres://schemahero:password@$(DATABASE_CONTAINER_NAME):5432/schemahero?sslmode=disable \ + --spec-file /specs/users.yaml + + # Verify + docker run \ + --rm \ + --network $(TEST_NAME) \ + -v `pwd`/out:/out \ + -e uid=$${UID} \ + schemahero/schemahero:test \ + generate \ + --dbname schemahero \ + --namespace default \ + --driver postgres \ + --output-dir /out \ + --uri postgres://schemahero:password@$(DATABASE_CONTAINER_NAME):5432/schemahero?sslmode=disable + @echo Verifying results for $(TEST_NAME) + diff --color expect out + + # Cleanup + @-sleep 5 + rm -rf ./out + @-docker rm -f $(DATABASE_CONTAINER_NAME) + @-docker network rm $(TEST_NAME) + diff --git a/integration/tests/postgres/create-table/expect/kustomization.yaml b/integration/tests/postgres/create-table/expect/kustomization.yaml new file mode 100644 index 000000000..93c9d3e20 --- /dev/null +++ b/integration/tests/postgres/create-table/expect/kustomization.yaml @@ -0,0 +1,3 @@ +resources: +- ./other.yaml +- ./users.yaml diff --git a/integration/tests/postgres/create-table/expect/other.yaml b/integration/tests/postgres/create-table/expect/other.yaml new file mode 100644 index 000000000..42541ff52 --- /dev/null +++ b/integration/tests/postgres/create-table/expect/other.yaml @@ -0,0 +1,20 @@ +apiVersion: schemas.schemahero.io/v1alpha1 +kind: Table +metadata: + name: other +spec: + database: schemahero + name: other + schema: + postgres: + primaryKey: + - id + columns: + - name: id + type: integer + constraints: + notNull: true + - name: something + type: character varying (255) + constraints: + notNull: true diff --git a/integration/tests/postgres/create-table/expect/users.yaml b/integration/tests/postgres/create-table/expect/users.yaml new file mode 100644 index 000000000..2242606ba --- /dev/null +++ b/integration/tests/postgres/create-table/expect/users.yaml @@ -0,0 +1,24 @@ +apiVersion: schemas.schemahero.io/v1alpha1 +kind: Table +metadata: + name: users +spec: + database: schemahero + name: users + schema: + postgres: + primaryKey: + - id + columns: + - name: id + type: integer + constraints: + notNull: true + - name: login + type: character varying (255) + constraints: + notNull: false + - name: name + type: character varying (255) + constraints: + notNull: false diff --git a/integration/tests/postgres/create-table/fixtures.sql b/integration/tests/postgres/create-table/fixtures.sql new file mode 100644 index 000000000..19b3b4010 --- /dev/null +++ b/integration/tests/postgres/create-table/fixtures.sql @@ -0,0 +1,4 @@ +create table other ( + id integer primary key not null, + something varchar(255) not null +); diff --git a/integration/tests/postgres/create-table/specs/users.yaml b/integration/tests/postgres/create-table/specs/users.yaml new file mode 100644 index 000000000..dbd3d0bf9 --- /dev/null +++ b/integration/tests/postgres/create-table/specs/users.yaml @@ -0,0 +1,13 @@ +database: schemahero +name: users +requires: [] +schema: + postgres: + primaryKey: [id] + columns: + - name: id + type: integer + - name: login + type: varchar(255) + - name: name + type: varchar(255) diff --git a/integration/tests/postgres/foreign-key-alter/Dockerfile b/integration/tests/postgres/foreign-key-alter/Dockerfile new file mode 100644 index 000000000..a0cc4ed89 --- /dev/null +++ b/integration/tests/postgres/foreign-key-alter/Dockerfile @@ -0,0 +1,10 @@ +FROM postgres:10.7 + +ENV POSTGRES_USER=schemahero +ENV POSTGRES_PASSWORD=password +ENV POSTGRES_DB=schemahero + +## Insert fixtures +COPY ./fixtures.sql /docker-entrypoint-initdb.d/ + + diff --git a/integration/tests/postgres/foreign-key-alter/Makefile b/integration/tests/postgres/foreign-key-alter/Makefile new file mode 100644 index 000000000..1bf19d16c --- /dev/null +++ b/integration/tests/postgres/foreign-key-alter/Makefile @@ -0,0 +1,55 @@ +SHELL := /bin/bash +TEST_NAME := postgres-foreign-key-alter +DATABASE_IMAGE_NAME := schemahero/database +DATABASE_CONTAINER_NAME := schemahero-database + +.PHONY: run +run: + @rm -rf ./out + @mkdir ./out + @-docker rm -f $(DATABASE_CONTAINER_NAME) > /dev/null 2>&1 ||: + @-docker rm -f $(TEST_NAME) > /dev/null 2>&1 ||: + @-docker network rm $(TEST_NAME) > /dev/null 2>&1 ||: + docker network create $(TEST_NAME) + + # Fixtures + docker pull postgres:10.7 + docker build -t $(DATABASE_IMAGE_NAME) . + @-docker rm -f $(DATABASE_CONTAINER_NAME) > /dev/null 2>&1 ||: + docker run --network $(TEST_NAME) --rm -d --name $(DATABASE_CONTAINER_NAME) $(DATABASE_IMAGE_NAME) + @-sleep 5 + + # Test + docker tag $(IMAGE) schemahero/schemahero:test + docker run -v `pwd`/specs:/specs \ + --network $(TEST_NAME) \ + --name $(TEST_NAME) \ + --rm \ + schemahero/schemahero:test \ + apply \ + --driver postgres \ + --uri postgres://schemahero:password@$(DATABASE_CONTAINER_NAME):5432/schemahero?sslmode=disable \ + --spec-file /specs/issues.yaml + + # Verify + docker run \ + --rm \ + --network $(TEST_NAME) \ + -v `pwd`/out:/out \ + -e uid=$${UID} \ + schemahero/schemahero:test \ + generate \ + --dbname schemahero \ + --namespace default \ + --driver postgres \ + --output-dir /out \ + --uri postgres://schemahero:password@$(DATABASE_CONTAINER_NAME):5432/schemahero?sslmode=disable + @echo Verifying results for $(TEST_NAME) + diff --color expect out + + # Cleanup + @-sleep 5 + rm -rf ./out + @-docker rm -f $(DATABASE_CONTAINER_NAME) + @-docker network rm $(TEST_NAME) + diff --git a/integration/tests/postgres/foreign-key-alter/expect/issues.yaml b/integration/tests/postgres/foreign-key-alter/expect/issues.yaml new file mode 100644 index 000000000..6b06a6f33 --- /dev/null +++ b/integration/tests/postgres/foreign-key-alter/expect/issues.yaml @@ -0,0 +1,28 @@ +apiVersion: schemas.schemahero.io/v1alpha1 +kind: Table +metadata: + name: issues +spec: + database: schemahero + name: issues + schema: + postgres: + primaryKey: + - id + foreignKeys: + - columns: + - project_id + references: + table: projects + columns: + - id + name: renamed_fkey + columns: + - name: id + type: integer + constraints: + notNull: true + - name: project_id + type: integer + constraints: + notNull: false diff --git a/integration/tests/postgres/foreign-key-alter/expect/kustomization.yaml b/integration/tests/postgres/foreign-key-alter/expect/kustomization.yaml new file mode 100644 index 000000000..c01451b77 --- /dev/null +++ b/integration/tests/postgres/foreign-key-alter/expect/kustomization.yaml @@ -0,0 +1,4 @@ +resources: +- ./users.yaml +- ./projects.yaml +- ./issues.yaml diff --git a/integration/tests/postgres/foreign-key-alter/expect/projects.yaml b/integration/tests/postgres/foreign-key-alter/expect/projects.yaml new file mode 100644 index 000000000..24d808a4d --- /dev/null +++ b/integration/tests/postgres/foreign-key-alter/expect/projects.yaml @@ -0,0 +1,20 @@ +apiVersion: schemas.schemahero.io/v1alpha1 +kind: Table +metadata: + name: projects +spec: + database: schemahero + name: projects + schema: + postgres: + primaryKey: + - id + columns: + - name: id + type: integer + constraints: + notNull: true + - name: name + type: character varying (255) + constraints: + notNull: true diff --git a/integration/tests/postgres/foreign-key-alter/expect/users.yaml b/integration/tests/postgres/foreign-key-alter/expect/users.yaml new file mode 100644 index 000000000..6ebf085d1 --- /dev/null +++ b/integration/tests/postgres/foreign-key-alter/expect/users.yaml @@ -0,0 +1,20 @@ +apiVersion: schemas.schemahero.io/v1alpha1 +kind: Table +metadata: + name: users +spec: + database: schemahero + name: users + schema: + postgres: + primaryKey: + - id + columns: + - name: id + type: integer + constraints: + notNull: true + - name: email + type: character varying (255) + constraints: + notNull: true diff --git a/integration/tests/postgres/foreign-key-alter/fixtures.sql b/integration/tests/postgres/foreign-key-alter/fixtures.sql new file mode 100644 index 000000000..f9f808d91 --- /dev/null +++ b/integration/tests/postgres/foreign-key-alter/fixtures.sql @@ -0,0 +1,14 @@ +create table users ( + id integer primary key not null, + email varchar(255) not null +); + +create table projects ( + id integer primary key not null, + name varchar(255) not null +); + +create table issues ( + id integer primary key not null, + project_id integer references users(id) +); diff --git a/integration/tests/postgres/foreign-key-alter/specs/issues.yaml b/integration/tests/postgres/foreign-key-alter/specs/issues.yaml new file mode 100644 index 000000000..f54ded51a --- /dev/null +++ b/integration/tests/postgres/foreign-key-alter/specs/issues.yaml @@ -0,0 +1,18 @@ +database: schemahero +name: issues +schema: + postgres: + primaryKey: [id] + foreignKeys: + - columns: + - project_id + references: + table: projects + columns: + - id + name: renamed_fkey + columns: + - name: id + type: integer + - name: project_id + type: integer diff --git a/integration/tests/postgres/foreign-key-create/Dockerfile b/integration/tests/postgres/foreign-key-create/Dockerfile new file mode 100644 index 000000000..a0cc4ed89 --- /dev/null +++ b/integration/tests/postgres/foreign-key-create/Dockerfile @@ -0,0 +1,10 @@ +FROM postgres:10.7 + +ENV POSTGRES_USER=schemahero +ENV POSTGRES_PASSWORD=password +ENV POSTGRES_DB=schemahero + +## Insert fixtures +COPY ./fixtures.sql /docker-entrypoint-initdb.d/ + + diff --git a/integration/tests/postgres/foreign-key-create/Makefile b/integration/tests/postgres/foreign-key-create/Makefile new file mode 100644 index 000000000..b11807396 --- /dev/null +++ b/integration/tests/postgres/foreign-key-create/Makefile @@ -0,0 +1,55 @@ +SHELL := /bin/bash +TEST_NAME := postgres-foreign-key-create +DATABASE_IMAGE_NAME := schemahero/database +DATABASE_CONTAINER_NAME := schemahero-database + +.PHONY: run +run: + @rm -rf ./out + @mkdir ./out + @-docker rm -f $(DATABASE_CONTAINER_NAME) > /dev/null 2>&1 ||: + @-docker rm -f $(TEST_NAME) > /dev/null 2>&1 ||: + @-docker network rm $(TEST_NAME) > /dev/null 2>&1 ||: + docker network create $(TEST_NAME) + + # Fixtures + docker pull postgres:10.7 + docker build -t $(DATABASE_IMAGE_NAME) . + @-docker rm -f $(DATABASE_CONTAINER_NAME) > /dev/null 2>&1 ||: + docker run --network $(TEST_NAME) --rm -d --name $(DATABASE_CONTAINER_NAME) $(DATABASE_IMAGE_NAME) + @-sleep 5 + + # Test + docker tag $(IMAGE) schemahero/schemahero:test + docker run -v `pwd`/specs:/specs \ + --network $(TEST_NAME) \ + --name $(TEST_NAME) \ + --rm \ + schemahero/schemahero:test \ + apply \ + --driver postgres \ + --uri postgres://schemahero:password@$(DATABASE_CONTAINER_NAME):5432/schemahero?sslmode=disable \ + --spec-file /specs/user-project.yaml + + # Verify + docker run \ + --rm \ + --network $(TEST_NAME) \ + -v `pwd`/out:/out \ + -e uid=$${UID} \ + schemahero/schemahero:test \ + generate \ + --dbname schemahero \ + --namespace default \ + --driver postgres \ + --output-dir /out \ + --uri postgres://schemahero:password@$(DATABASE_CONTAINER_NAME):5432/schemahero?sslmode=disable + @echo Verifying results for $(TEST_NAME) + diff --color expect out + + # Cleanup + @-sleep 5 + rm -rf ./out + @-docker rm -f $(DATABASE_CONTAINER_NAME) + @-docker network rm $(TEST_NAME) + diff --git a/integration/tests/postgres/foreign-key-create/expect/kustomization.yaml b/integration/tests/postgres/foreign-key-create/expect/kustomization.yaml new file mode 100644 index 000000000..5e8f24513 --- /dev/null +++ b/integration/tests/postgres/foreign-key-create/expect/kustomization.yaml @@ -0,0 +1,5 @@ +resources: +- ./users.yaml +- ./projects.yaml +- ./misc.yaml +- ./user-project.yaml diff --git a/integration/tests/postgres/foreign-key-create/expect/misc.yaml b/integration/tests/postgres/foreign-key-create/expect/misc.yaml new file mode 100644 index 000000000..0eab18b26 --- /dev/null +++ b/integration/tests/postgres/foreign-key-create/expect/misc.yaml @@ -0,0 +1,16 @@ +apiVersion: schemas.schemahero.io/v1alpha1 +kind: Table +metadata: + name: misc +spec: + database: schemahero + name: misc + schema: + postgres: + primaryKey: + - pk + columns: + - name: pk + type: character varying (255) + constraints: + notNull: true diff --git a/integration/tests/postgres/foreign-key-create/expect/projects.yaml b/integration/tests/postgres/foreign-key-create/expect/projects.yaml new file mode 100644 index 000000000..24d808a4d --- /dev/null +++ b/integration/tests/postgres/foreign-key-create/expect/projects.yaml @@ -0,0 +1,20 @@ +apiVersion: schemas.schemahero.io/v1alpha1 +kind: Table +metadata: + name: projects +spec: + database: schemahero + name: projects + schema: + postgres: + primaryKey: + - id + columns: + - name: id + type: integer + constraints: + notNull: true + - name: name + type: character varying (255) + constraints: + notNull: true diff --git a/integration/tests/postgres/foreign-key-create/expect/user-project.yaml b/integration/tests/postgres/foreign-key-create/expect/user-project.yaml new file mode 100644 index 000000000..8769f121d --- /dev/null +++ b/integration/tests/postgres/foreign-key-create/expect/user-project.yaml @@ -0,0 +1,47 @@ +apiVersion: schemas.schemahero.io/v1alpha1 +kind: Table +metadata: + name: user-project +spec: + database: schemahero + name: user_project + schema: + postgres: + primaryKey: + - user_id + - project_id + foreignKeys: + - columns: + - user_id + references: + table: users + columns: + - id + name: user_project_user_id_fkey + - columns: + - project_id + references: + table: projects + columns: + - id + name: user_project_project_id_fkey + - columns: + - misc_id + references: + table: misc + columns: + - pk + name: misc_named_fk + columns: + - name: user_id + type: integer + constraints: + notNull: true + - name: project_id + type: integer + constraints: + notNull: true + - name: misc_id + type: character varying (255) + constraints: + notNull: true diff --git a/integration/tests/postgres/foreign-key-create/expect/users.yaml b/integration/tests/postgres/foreign-key-create/expect/users.yaml new file mode 100644 index 000000000..6ebf085d1 --- /dev/null +++ b/integration/tests/postgres/foreign-key-create/expect/users.yaml @@ -0,0 +1,20 @@ +apiVersion: schemas.schemahero.io/v1alpha1 +kind: Table +metadata: + name: users +spec: + database: schemahero + name: users + schema: + postgres: + primaryKey: + - id + columns: + - name: id + type: integer + constraints: + notNull: true + - name: email + type: character varying (255) + constraints: + notNull: true diff --git a/integration/tests/postgres/foreign-key-create/fixtures.sql b/integration/tests/postgres/foreign-key-create/fixtures.sql new file mode 100644 index 000000000..1d73f1726 --- /dev/null +++ b/integration/tests/postgres/foreign-key-create/fixtures.sql @@ -0,0 +1,14 @@ +create table users ( + id integer primary key not null, + email varchar(255) not null +); + +create table projects ( + id integer primary key not null, + name varchar(255) not null +); + +create table misc ( + pk varchar(255) primary key not null +); + diff --git a/integration/tests/postgres/foreign-key-create/specs/user-project.yaml b/integration/tests/postgres/foreign-key-create/specs/user-project.yaml new file mode 100644 index 000000000..cc9513650 --- /dev/null +++ b/integration/tests/postgres/foreign-key-create/specs/user-project.yaml @@ -0,0 +1,34 @@ +database: schemahero +name: user_project +schema: + postgres: + primaryKey: [user_id, project_id] + foreignKeys: + - columns: + - user_id + references: + table: users + columns: + - id + - columns: + - project_id + references: + table: projects + columns: + - id + - columns: + - misc_id + references: + table: misc + columns: + - pk + name: misc_named_fk + columns: + - name: user_id + type: integer + - name: project_id + type: integer + - name: misc_id + type: varchar(255) + constraints: + notNull: true diff --git a/integration/tests/postgres/foreign-key-drop/Dockerfile b/integration/tests/postgres/foreign-key-drop/Dockerfile new file mode 100644 index 000000000..a0cc4ed89 --- /dev/null +++ b/integration/tests/postgres/foreign-key-drop/Dockerfile @@ -0,0 +1,10 @@ +FROM postgres:10.7 + +ENV POSTGRES_USER=schemahero +ENV POSTGRES_PASSWORD=password +ENV POSTGRES_DB=schemahero + +## Insert fixtures +COPY ./fixtures.sql /docker-entrypoint-initdb.d/ + + diff --git a/integration/tests/postgres/foreign-key-drop/Makefile b/integration/tests/postgres/foreign-key-drop/Makefile new file mode 100644 index 000000000..3bf5d5259 --- /dev/null +++ b/integration/tests/postgres/foreign-key-drop/Makefile @@ -0,0 +1,55 @@ +SHELL := /bin/bash +TEST_NAME := postgres-foreign-key-drop +DATABASE_IMAGE_NAME := schemahero/database +DATABASE_CONTAINER_NAME := schemahero-database + +.PHONY: run +run: + @rm -rf ./out + @mkdir ./out + @-docker rm -f $(DATABASE_CONTAINER_NAME) > /dev/null 2>&1 ||: + @-docker rm -f $(TEST_NAME) > /dev/null 2>&1 ||: + @-docker network rm $(TEST_NAME) > /dev/null 2>&1 ||: + docker network create $(TEST_NAME) + + # Fixtures + docker pull postgres:10.7 + docker build -t $(DATABASE_IMAGE_NAME) . + @-docker rm -f $(DATABASE_CONTAINER_NAME) > /dev/null 2>&1 ||: + docker run --network $(TEST_NAME) --rm -d --name $(DATABASE_CONTAINER_NAME) $(DATABASE_IMAGE_NAME) + @-sleep 5 + + # Test + docker tag $(IMAGE) schemahero/schemahero:test + docker run -v `pwd`/specs:/specs \ + --network $(TEST_NAME) \ + --name $(TEST_NAME) \ + --rm \ + schemahero/schemahero:test \ + apply \ + --driver postgres \ + --uri postgres://schemahero:password@$(DATABASE_CONTAINER_NAME):5432/schemahero?sslmode=disable \ + --spec-file /specs/org.yaml + + # Verify + docker run \ + --rm \ + --network $(TEST_NAME) \ + -v `pwd`/out:/out \ + -e uid=$${UID} \ + schemahero/schemahero:test \ + generate \ + --dbname schemahero \ + --namespace default \ + --driver postgres \ + --output-dir /out \ + --uri postgres://schemahero:password@$(DATABASE_CONTAINER_NAME):5432/schemahero?sslmode=disable + @echo Verifying results for $(TEST_NAME) + diff --color expect out + + # Cleanup + @-sleep 5 + rm -rf ./out + @-docker rm -f $(DATABASE_CONTAINER_NAME) + @-docker network rm $(TEST_NAME) + diff --git a/integration/tests/postgres/foreign-key-drop/expect/kustomization.yaml b/integration/tests/postgres/foreign-key-drop/expect/kustomization.yaml new file mode 100644 index 000000000..3968fb157 --- /dev/null +++ b/integration/tests/postgres/foreign-key-drop/expect/kustomization.yaml @@ -0,0 +1,3 @@ +resources: +- ./projects.yaml +- ./org.yaml diff --git a/integration/tests/postgres/foreign-key-drop/expect/org.yaml b/integration/tests/postgres/foreign-key-drop/expect/org.yaml new file mode 100644 index 000000000..80aa1a8c2 --- /dev/null +++ b/integration/tests/postgres/foreign-key-drop/expect/org.yaml @@ -0,0 +1,20 @@ +apiVersion: schemas.schemahero.io/v1alpha1 +kind: Table +metadata: + name: org +spec: + database: schemahero + name: org + schema: + postgres: + primaryKey: + - id + columns: + - name: id + type: integer + constraints: + notNull: true + - name: project_id + type: integer + constraints: + notNull: true diff --git a/integration/tests/postgres/foreign-key-drop/expect/projects.yaml b/integration/tests/postgres/foreign-key-drop/expect/projects.yaml new file mode 100644 index 000000000..24d808a4d --- /dev/null +++ b/integration/tests/postgres/foreign-key-drop/expect/projects.yaml @@ -0,0 +1,20 @@ +apiVersion: schemas.schemahero.io/v1alpha1 +kind: Table +metadata: + name: projects +spec: + database: schemahero + name: projects + schema: + postgres: + primaryKey: + - id + columns: + - name: id + type: integer + constraints: + notNull: true + - name: name + type: character varying (255) + constraints: + notNull: true diff --git a/integration/tests/postgres/foreign-key-drop/fixtures.sql b/integration/tests/postgres/foreign-key-drop/fixtures.sql new file mode 100644 index 000000000..aa461b4a2 --- /dev/null +++ b/integration/tests/postgres/foreign-key-drop/fixtures.sql @@ -0,0 +1,9 @@ +create table projects ( + id integer primary key not null, + name varchar(255) not null +); + +create table org ( + id integer primary key not null, + project_id integer references projects(id) +); diff --git a/integration/tests/postgres/foreign-key-drop/specs/org.yaml b/integration/tests/postgres/foreign-key-drop/specs/org.yaml new file mode 100644 index 000000000..87e2a5a98 --- /dev/null +++ b/integration/tests/postgres/foreign-key-drop/specs/org.yaml @@ -0,0 +1,15 @@ +database: schemahero +name: org +schema: + postgres: + primaryKey: + - id + columns: + - name: id + type: integer + constraints: + notNull: true + - name: project_id + type: integer + constraints: + notNull: true diff --git a/pkg/apis/databases/v1alpha1/mysql_types.test.go b/pkg/apis/databases/v1alpha1/mysql_types_test.go similarity index 100% rename from pkg/apis/databases/v1alpha1/mysql_types.test.go rename to pkg/apis/databases/v1alpha1/mysql_types_test.go diff --git a/pkg/apis/schemas/v1alpha1/sql.go b/pkg/apis/schemas/v1alpha1/sql.go index 2d3e570ba..1c1b549a7 100644 --- a/pkg/apis/schemas/v1alpha1/sql.go +++ b/pkg/apis/schemas/v1alpha1/sql.go @@ -16,19 +16,31 @@ limitations under the License. package v1alpha1 +type SQLTableForeignKeyReferences struct { + Table string `json:"table"` + Columns []string `json:"columns"` +} + +type SQLTableForeignKey struct { + Columns []string `json:"columns" yaml:"columns"` + References SQLTableForeignKeyReferences `json:"references" yaml:"references"` + Name string `json:"name,omitempty" yaml:"name,omitempty"` +} + type SQLTableColumnConstraints struct { NotNull *bool `json:"notNull,omitempty" yaml:"notNull,omitempty"` } type SQLTableColumn struct { - Name string `json:"name"` - Type string `json:"type"` - Constraints *SQLTableColumnConstraints `json:"constraints,omitempty"` - Default string `json:"default,omitempty"` + Name string `json:"name" yaml:"name"` + Type string `json:"type" yaml:"type"` + Constraints *SQLTableColumnConstraints `json:"constraints,omitempty" yaml:"constraints,omitempty"` + Default string `json:"default,omitempty" yaml:"defualt,omitempty"` } type SQLTableSchema struct { - PrimaryKey []string `json:"primaryKey" yaml:"primaryKey"` - Columns []*SQLTableColumn `json:"columns,omitempty"` - IsDeleted bool `json:"isDeleted,omitempty"` + PrimaryKey []string `json:"primaryKey" yaml:"primaryKey"` + ForeignKeys []*SQLTableForeignKey `json:"foreignKeys,omitempty" yaml:"foreignKeys,omitempty"` + Columns []*SQLTableColumn `json:"columns,omitempty" yaml:"columns"` + IsDeleted bool `json:"isDeleted,omitempty" yaml:"isDeleted,omitempty"` } diff --git a/pkg/apis/schemas/v1alpha1/sql_types_test.go b/pkg/apis/schemas/v1alpha1/sql_types_test.go new file mode 100644 index 000000000..649af2b24 --- /dev/null +++ b/pkg/apis/schemas/v1alpha1/sql_types_test.go @@ -0,0 +1,60 @@ +/* +Copyright 2019 Replicated, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + "testing" + + "github.com/onsi/gomega" + "gopkg.in/yaml.v2" +) + +func Test_SQLTypes(t *testing.T) { + const fk = ` +isDeleted: false +columns: + - name: id + type: integer + - name: order_id + type: integer +primaryKey: + - id +foreignKeys: + - columns: + - order_id + references: + table: order + columns: + - id +` + + g := gomega.NewGomegaWithT(t) + + table := SQLTableSchema{} + err := yaml.Unmarshal([]byte(fk), &table) + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(table.Columns).To(gomega.HaveLen(2)) + g.Expect(table.ForeignKeys).To(gomega.HaveLen(1)) + + g.Expect(table.ForeignKeys[0].Columns).To(gomega.HaveLen(1)) + g.Expect(table.ForeignKeys[0].Columns[0]).To(gomega.Equal("order_id")) + + g.Expect(table.ForeignKeys[0].References.Table).To(gomega.Equal("order")) + g.Expect(table.ForeignKeys[0].References.Columns).To(gomega.HaveLen(1)) + g.Expect(table.ForeignKeys[0].References.Columns[0]).To(gomega.Equal("id")) + +} diff --git a/pkg/apis/schemas/v1alpha1/table_types.go b/pkg/apis/schemas/v1alpha1/table_types.go index 23b6c29ae..1ffe641a1 100644 --- a/pkg/apis/schemas/v1alpha1/table_types.go +++ b/pkg/apis/schemas/v1alpha1/table_types.go @@ -21,16 +21,16 @@ import ( ) type TableSchema struct { - Postgres *SQLTableSchema `json:"postgres,omitempty"` - Mysql *SQLTableSchema `json:"mysql,omitempty"` + Postgres *SQLTableSchema `json:"postgres,omitempty" yaml:"postgres,omitempty"` + Mysql *SQLTableSchema `json:"mysql,omitempty" yaml:"mysql,omitempty"` } // TableSpec defines the desired state of Table type TableSpec struct { - Database string `json:"database"` - Name string `json:"name"` - Requires []string `json:"requires"` - Schema *TableSchema `json:"schema"` + Database string `json:"database" yaml:"database"` + Name string `json:"name" yaml:"name"` + Requires []string `json:"requires,omitempty" yaml:"requires,omitempty"` + Schema *TableSchema `json:"schema" yaml:"schema"` } // TableStatus defines the observed state of Table diff --git a/pkg/apis/schemas/v1alpha1/table_types_test.go b/pkg/apis/schemas/v1alpha1/table_types_test.go index 29016c666..61caf0e2d 100644 --- a/pkg/apis/schemas/v1alpha1/table_types_test.go +++ b/pkg/apis/schemas/v1alpha1/table_types_test.go @@ -44,7 +44,6 @@ func TestStorageTable(t *testing.T) { Spec: TableSpec{ Database: "d", Name: "n", - Requires: []string{}, Schema: &TableSchema{ Postgres: &SQLTableSchema{ PrimaryKey: []string{"pk"}, diff --git a/pkg/apis/schemas/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/schemas/v1alpha1/zz_generated.deepcopy.go index bbe472428..8d12939a0 100644 --- a/pkg/apis/schemas/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/schemas/v1alpha1/zz_generated.deepcopy.go @@ -158,6 +158,49 @@ func (in *SQLTableColumnConstraints) DeepCopy() *SQLTableColumnConstraints { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SQLTableForeignKey) DeepCopyInto(out *SQLTableForeignKey) { + *out = *in + if in.Columns != nil { + in, out := &in.Columns, &out.Columns + *out = make([]string, len(*in)) + copy(*out, *in) + } + in.References.DeepCopyInto(&out.References) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SQLTableForeignKey. +func (in *SQLTableForeignKey) DeepCopy() *SQLTableForeignKey { + if in == nil { + return nil + } + out := new(SQLTableForeignKey) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SQLTableForeignKeyReferences) DeepCopyInto(out *SQLTableForeignKeyReferences) { + *out = *in + if in.Columns != nil { + in, out := &in.Columns, &out.Columns + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SQLTableForeignKeyReferences. +func (in *SQLTableForeignKeyReferences) DeepCopy() *SQLTableForeignKeyReferences { + if in == nil { + return nil + } + out := new(SQLTableForeignKeyReferences) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SQLTableSchema) DeepCopyInto(out *SQLTableSchema) { *out = *in @@ -166,6 +209,17 @@ func (in *SQLTableSchema) DeepCopyInto(out *SQLTableSchema) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.ForeignKeys != nil { + in, out := &in.ForeignKeys, &out.ForeignKeys + *out = make([]*SQLTableForeignKey, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(SQLTableForeignKey) + (*in).DeepCopyInto(*out) + } + } + } if in.Columns != nil { in, out := &in.Columns, &out.Columns *out = make([]*SQLTableColumn, len(*in)) diff --git a/pkg/database/database.go b/pkg/database/database.go index 83709995e..4b17c0955 100644 --- a/pkg/database/database.go +++ b/pkg/database/database.go @@ -64,8 +64,20 @@ func (d *Database) CreateFixturesSync() error { return err } + statements = append(statements, statement) + } else if d.Viper.GetString("driver") == "mysql" { + if parsed.Spec.Schema.Mysql == nil { + fmt.Printf("skipping file %s because there is no mysql spec\n", info.Name()) + } + + statement, err := mysql.CreateTableStatement(parsed.Spec.Name, parsed.Spec.Schema.Mysql) + if err != nil { + return err + } + statements = append(statements, statement) } + return nil } diff --git a/pkg/database/interfaces/connection.go b/pkg/database/interfaces/connection.go index 3cbecf92c..ef0d4e5b1 100644 --- a/pkg/database/interfaces/connection.go +++ b/pkg/database/interfaces/connection.go @@ -1,6 +1,10 @@ package interfaces -import "database/sql" +import ( + "database/sql" + + "github.com/schemahero/schemahero/pkg/database/types" +) type SchemaHeroDatabaseConnection interface { GetConnection() *sql.Conn @@ -10,4 +14,10 @@ type SchemaHeroDatabaseConnection interface { EngineVersion() string CheckAlive(string, string) (bool, error) + + ListTables() ([]string, error) + ListTableForeignKeys(string, string) ([]*types.ForeignKey, error) + + GetTablePrimaryKey(string) ([]string, error) + GetTableSchema(string) ([]*types.Column, error) } diff --git a/pkg/database/mysql/alter.go b/pkg/database/mysql/alter.go index 12e856397..2c1054116 100644 --- a/pkg/database/mysql/alter.go +++ b/pkg/database/mysql/alter.go @@ -5,9 +5,10 @@ import ( "strings" schemasv1alpha1 "github.com/schemahero/schemahero/pkg/apis/schemas/v1alpha1" + "github.com/schemahero/schemahero/pkg/database/types" ) -func columnsMatch(col1 *Column, col2 *Column) bool { +func columnsMatch(col1 *types.Column, col2 *types.Column) bool { if col1.DataType != col2.DataType { return false } @@ -29,12 +30,12 @@ func columnsMatch(col1 *Column, col2 *Column) bool { return true } -func AlterColumnStatement(tableName string, desiredColumns []*schemasv1alpha1.SQLTableColumn, existingColumn *Column) (string, error) { +func AlterColumnStatement(tableName string, desiredColumns []*schemasv1alpha1.SQLTableColumn, existingColumn *types.Column) (string, error) { // this could be an alter or a drop column command columnStatement := "" for _, desiredColumn := range desiredColumns { if desiredColumn.Name == existingColumn.Name { - column, err := schemaColumnToMysqlColumn(desiredColumn) + column, err := schemaColumnToColumn(desiredColumn) if err != nil { return "", err } @@ -45,7 +46,7 @@ func AlterColumnStatement(tableName string, desiredColumns []*schemasv1alpha1.SQ changes := []string{} if existingColumn.DataType != column.DataType { - changes = append(changes, fmt.Sprintf("%s type %s", columnStatement, column.DataType)) + changes = append(changes, fmt.Sprintf("%s %s", columnStatement, column.DataType)) } // too much complexity below! @@ -54,7 +55,7 @@ func AlterColumnStatement(tableName string, desiredColumns []*schemasv1alpha1.SQ if column.Constraints != nil && column.Constraints.NotNull != nil && *column.Constraints.NotNull == true { if existingColumn.Constraints != nil || existingColumn.Constraints.NotNull != nil { if *existingColumn.Constraints.NotNull == false { - changes = append(changes, fmt.Sprintf("%s set not null", columnStatement)) + changes = append(changes, fmt.Sprintf("%s not null", columnStatement)) } } } @@ -63,7 +64,7 @@ func AlterColumnStatement(tableName string, desiredColumns []*schemasv1alpha1.SQ if column.Constraints != nil && column.Constraints.NotNull != nil && *column.Constraints.NotNull == false { if existingColumn.Constraints != nil || existingColumn.Constraints.NotNull != nil { if *existingColumn.Constraints.NotNull == true { - changes = append(changes, fmt.Sprintf("%s drop not null", columnStatement)) + changes = append(changes, fmt.Sprintf("%s null", columnStatement)) } } } @@ -74,7 +75,7 @@ func AlterColumnStatement(tableName string, desiredColumns []*schemasv1alpha1.SQ return "", nil } - columnStatement = fmt.Sprintf("alter table `%s` alter column `%s`%s", tableName, existingColumn.Name, strings.Join(changes, " ")) + columnStatement = fmt.Sprintf("alter table `%s` modify column `%s`%s", tableName, existingColumn.Name, strings.Join(changes, " ")) } } diff --git a/pkg/database/mysql/alter_test.go b/pkg/database/mysql/alter_test.go index c960fe8a0..ffc2d4ba6 100644 --- a/pkg/database/mysql/alter_test.go +++ b/pkg/database/mysql/alter_test.go @@ -4,6 +4,7 @@ import ( "testing" schemasv1alpha1 "github.com/schemahero/schemahero/pkg/apis/schemas/v1alpha1" + "github.com/schemahero/schemahero/pkg/database/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -14,7 +15,7 @@ func Test_AlterColumnStatment(t *testing.T) { name string tableName string desiredColumns []*schemasv1alpha1.SQLTableColumn - existingColumn *Column + existingColumn *types.Column expectedStatement string }{ { @@ -30,7 +31,7 @@ func Test_AlterColumnStatment(t *testing.T) { Type: "integer", }, }, - existingColumn: &Column{ + existingColumn: &types.Column{ Name: "b", DataType: "int (11)", ColumnDefault: nil, @@ -50,12 +51,12 @@ func Test_AlterColumnStatment(t *testing.T) { Type: "integer", }, }, - existingColumn: &Column{ + existingColumn: &types.Column{ Name: "b", DataType: "varchar(255)", ColumnDefault: nil, }, - expectedStatement: "alter table `t` alter column `b` type int (11)", + expectedStatement: "alter table `t` modify column `b` int (11)", }, { name: "drop column", @@ -66,7 +67,7 @@ func Test_AlterColumnStatment(t *testing.T) { Type: "integer", }, }, - existingColumn: &Column{ + existingColumn: &types.Column{ Name: "b", DataType: "varchar (255)", ColumnDefault: nil, @@ -85,15 +86,15 @@ func Test_AlterColumnStatment(t *testing.T) { }, }, }, - existingColumn: &Column{ + existingColumn: &types.Column{ Name: "a", DataType: "int (11)", ColumnDefault: nil, - Constraints: &ColumnConstraints{ + Constraints: &types.ColumnConstraints{ NotNull: &falseValue, }, }, - expectedStatement: "alter table `t` alter column `a` set not null", + expectedStatement: "alter table `t` modify column `a` not null", }, { name: "drop not null constraint", @@ -107,15 +108,15 @@ func Test_AlterColumnStatment(t *testing.T) { }, }, }, - existingColumn: &Column{ + existingColumn: &types.Column{ Name: "a", DataType: "int (11)", ColumnDefault: nil, - Constraints: &ColumnConstraints{ + Constraints: &types.ColumnConstraints{ NotNull: &trueValue, }, }, - expectedStatement: "alter table `t` alter column `a` drop not null", + expectedStatement: "alter table `t` modify column `a` null", }, { name: "no change to not null constraint", @@ -126,11 +127,11 @@ func Test_AlterColumnStatment(t *testing.T) { Type: "text", }, }, - existingColumn: &Column{ + existingColumn: &types.Column{ Name: "t", DataType: "text", ColumnDefault: nil, - Constraints: &ColumnConstraints{ + Constraints: &types.ColumnConstraints{ NotNull: &falseValue, }, }, diff --git a/pkg/database/mysql/column.go b/pkg/database/mysql/column.go index 27a7d2d28..7e3cab2e3 100644 --- a/pkg/database/mysql/column.go +++ b/pkg/database/mysql/column.go @@ -4,42 +4,14 @@ import ( "fmt" schemasv1alpha1 "github.com/schemahero/schemahero/pkg/apis/schemas/v1alpha1" + "github.com/schemahero/schemahero/pkg/database/types" ) -type ColumnConstraints struct { - NotNull *bool -} - -type Column struct { - Name string - DataType string - ColumnDefault *string - Constraints *ColumnConstraints -} - -func MysqlColumnToSchemaColumn(column *Column) (*schemasv1alpha1.SQLTableColumn, error) { - constraints := &schemasv1alpha1.SQLTableColumnConstraints{ - NotNull: column.Constraints.NotNull, - } - - schemaColumn := &schemasv1alpha1.SQLTableColumn{ - Name: column.Name, - Type: column.DataType, - Constraints: constraints, - } - - if column.ColumnDefault != nil { - schemaColumn.Default = *column.ColumnDefault - } - - return schemaColumn, nil -} - -func schemaColumnToMysqlColumn(schemaColumn *schemasv1alpha1.SQLTableColumn) (*Column, error) { - column := &Column{} +func schemaColumnToColumn(schemaColumn *schemasv1alpha1.SQLTableColumn) (*types.Column, error) { + column := &types.Column{} if schemaColumn.Constraints != nil { - column.Constraints = &ColumnConstraints{ + column.Constraints = &types.ColumnConstraints{ NotNull: schemaColumn.Constraints.NotNull, } } @@ -74,7 +46,7 @@ func schemaColumnToMysqlColumn(schemaColumn *schemasv1alpha1.SQLTableColumn) (*C } func mysqlColumnAsInsert(column *schemasv1alpha1.SQLTableColumn) (string, error) { - mysqlColumn, err := schemaColumnToMysqlColumn(column) + mysqlColumn, err := schemaColumnToColumn(column) if err != nil { return "", err } diff --git a/pkg/database/mysql/column_test.go b/pkg/database/mysql/column_test.go index faa25051a..c994b4d0e 100644 --- a/pkg/database/mysql/column_test.go +++ b/pkg/database/mysql/column_test.go @@ -4,6 +4,7 @@ import ( "testing" schemasv1alpha1 "github.com/schemahero/schemahero/pkg/apis/schemas/v1alpha1" + "github.com/schemahero/schemahero/pkg/database/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -93,7 +94,7 @@ func Test_schemaColumnToMysqlColumn(t *testing.T) { tests := []struct { name string schemaColumn *schemasv1alpha1.SQLTableColumn - expectedColumn *Column + expectedColumn *types.Column }{ { name: "varchar (10)", @@ -101,7 +102,7 @@ func Test_schemaColumnToMysqlColumn(t *testing.T) { Name: "vc", Type: "varchar (10)", }, - expectedColumn: &Column{ + expectedColumn: &types.Column{ DataType: "varchar (10)", ColumnDefault: nil, }, @@ -112,7 +113,7 @@ func Test_schemaColumnToMysqlColumn(t *testing.T) { Name: "b", Type: "bool", }, - expectedColumn: &Column{ + expectedColumn: &types.Column{ DataType: "tinyint (1)", ColumnDefault: nil, }, @@ -123,7 +124,7 @@ func Test_schemaColumnToMysqlColumn(t *testing.T) { t.Run(test.name, func(t *testing.T) { req := require.New(t) - column, err := schemaColumnToMysqlColumn(test.schemaColumn) + column, err := schemaColumnToColumn(test.schemaColumn) req.NoError(err) assert.Equal(t, test.expectedColumn, column) }) diff --git a/pkg/database/mysql/create.go b/pkg/database/mysql/create.go index 7e00922c3..dba855822 100644 --- a/pkg/database/mysql/create.go +++ b/pkg/database/mysql/create.go @@ -26,6 +26,13 @@ func CreateTableStatement(tableName string, tableSchema *schemasv1alpha1.SQLTabl columns = append(columns, fmt.Sprintf("primary key (%s)", strings.Join(primaryKeyColumns, ", "))) } + if tableSchema.ForeignKeys != nil { + for _, foreignKey := range tableSchema.ForeignKeys { + columns = append(columns, fmt.Sprintf("foreign key (%s) references %s (%s)", strings.Join(foreignKey.Columns, ", "), foreignKey.References.Table, strings.Join(foreignKey.References.Columns, ", "))) + } + + } + query := fmt.Sprintf("create table `%s` (%s)", tableName, strings.Join(columns, ", ")) return query, nil diff --git a/pkg/database/mysql/deploy.go b/pkg/database/mysql/deploy.go index d3f93f676..d07dbd4bb 100644 --- a/pkg/database/mysql/deploy.go +++ b/pkg/database/mysql/deploy.go @@ -6,24 +6,20 @@ import ( "fmt" schemasv1alpha1 "github.com/schemahero/schemahero/pkg/apis/schemas/v1alpha1" + "github.com/schemahero/schemahero/pkg/database/types" ) func DeployMysqlTable(uri string, tableName string, mysqlTableSchema *schemasv1alpha1.SQLTableSchema) error { - db, err := sql.Open("mysql", uri) - if err != nil { - return err - } - defer db.Close() - - databaseName, err := DatabaseNameFromURI(uri) + m, err := Connect(uri) if err != nil { return err } + defer m.db.Close() // determine if the table exists query := `select count(1) from information_schema.TABLES where TABLE_NAME = ? and TABLE_SCHEMA = ?` fmt.Printf("Executing query %q\n", query) - row := db.QueryRow(query, tableName, databaseName) + row := m.db.QueryRow(query, tableName, m.databaseName) tableExists := 0 if err := row.Scan(&tableExists); err != nil { return err @@ -37,7 +33,7 @@ func DeployMysqlTable(uri string, tableName string, mysqlTableSchema *schemasv1a } fmt.Printf("Executing query %q\n", query) - _, err = db.Exec(query) + _, err = m.db.Exec(query) if err != nil { return err } @@ -51,7 +47,7 @@ func DeployMysqlTable(uri string, tableName string, mysqlTableSchema *schemasv1a from information_schema.COLUMNS where TABLE_NAME = ?` fmt.Printf("Executing query %q\n", query) - rows, err := db.Query(query, tableName) + rows, err := m.db.Query(query, tableName) if err != nil { return err } @@ -62,17 +58,16 @@ func DeployMysqlTable(uri string, tableName string, mysqlTableSchema *schemasv1a var columnDefault sql.NullString var charMaxLength sql.NullInt64 - if err := rows.Scan(&columnName, &columnDefault, &isNullable, &dataType, - &charMaxLength); err != nil { + if err := rows.Scan(&columnName, &columnDefault, &isNullable, &dataType, &charMaxLength); err != nil { return err } foundColumnNames = append(foundColumnNames, columnName) - existingColumn := Column{ + existingColumn := types.Column{ Name: columnName, DataType: dataType, - Constraints: &ColumnConstraints{}, + Constraints: &types.ColumnConstraints{}, } if isNullable == "NO" { @@ -114,9 +109,65 @@ func DeployMysqlTable(uri string, tableName string, mysqlTableSchema *schemasv1a } } + // foreign key changes + currentForeignKeys, err := m.ListTableForeignKeys(m.databaseName, tableName) + if err != nil { + return err + } + for _, foreignKey := range mysqlTableSchema.ForeignKeys { + var statement string + var err error + + var matchedForeignKey *types.ForeignKey + for _, currentForeignKey := range currentForeignKeys { + if currentForeignKey.Equals(types.SchemaForeignKeyToForeignKey(foreignKey)) { + goto Next + } + + matchedForeignKey = currentForeignKey + } + + // drop and readd? is this always ok + // TODO can we alter + if matchedForeignKey != nil { + statement, err = RemoveForeignKeyStatement(tableName, matchedForeignKey) + if err != nil { + return err + } + alterAndDropStatements = append(alterAndDropStatements, statement) + } + + statement, err = AddForeignKeyStatement(tableName, foreignKey) + if err != nil { + return err + } + alterAndDropStatements = append(alterAndDropStatements, statement) + + Next: + } + + for _, currentForeignKey := range currentForeignKeys { + var statement string + var err error + + for _, foreignKey := range mysqlTableSchema.ForeignKeys { + if currentForeignKey.Equals(types.SchemaForeignKeyToForeignKey(foreignKey)) { + goto NextCurrentFK + } + } + + statement, err = RemoveForeignKeyStatement(tableName, currentForeignKey) + if err != nil { + return err + } + alterAndDropStatements = append(alterAndDropStatements, statement) + + NextCurrentFK: + } + for _, alterOrDropStatement := range alterAndDropStatements { fmt.Printf("Executing query %q\n", alterOrDropStatement) - if _, err = db.ExecContext(context.Background(), alterOrDropStatement); err != nil { + if _, err = m.db.ExecContext(context.Background(), alterOrDropStatement); err != nil { return err } } diff --git a/pkg/database/mysql/foreignkey.go b/pkg/database/mysql/foreignkey.go new file mode 100644 index 000000000..cb15b4943 --- /dev/null +++ b/pkg/database/mysql/foreignkey.go @@ -0,0 +1,23 @@ +package mysql + +import ( + "fmt" + "strings" + + schemasv1alpha1 "github.com/schemahero/schemahero/pkg/apis/schemas/v1alpha1" + "github.com/schemahero/schemahero/pkg/database/types" +) + +func RemoveForeignKeyStatement(tableName string, foreignKey *types.ForeignKey) (string, error) { + return fmt.Sprintf("alter table %s drop constraint %s", tableName, foreignKey.Name), nil +} + +func AddForeignKeyStatement(tableName string, schemaForeignKey *schemasv1alpha1.SQLTableForeignKey) (string, error) { + return fmt.Sprintf("alter table %s add constraint %s foreign key (%s) references %s (%s)", + tableName, + types.GenerateFKName(tableName, schemaForeignKey), + strings.Join(schemaForeignKey.Columns, ", "), + schemaForeignKey.References.Table, + strings.Join(schemaForeignKey.References.Columns, ", ")), + nil +} diff --git a/pkg/database/mysql/tables.go b/pkg/database/mysql/tables.go new file mode 100644 index 000000000..1b5793b9c --- /dev/null +++ b/pkg/database/mysql/tables.go @@ -0,0 +1,148 @@ +package mysql + +import ( + "database/sql" + "fmt" + + "github.com/schemahero/schemahero/pkg/database/types" +) + +func (m *MysqlConnection) ListTables() ([]string, error) { + query := "select table_name from information_schema.TABLES where TABLE_SCHEMA = ?" + + rows, err := m.db.Query(query, m.databaseName) + if err != nil { + return nil, err + } + + tableNames := make([]string, 0, 0) + for rows.Next() { + tableName := "" + if err := rows.Scan(&tableName); err != nil { + return nil, err + } + + tableNames = append(tableNames, tableName) + } + + return tableNames, nil +} + +func (m *MysqlConnection) ListTableForeignKeys(databaseName string, tableName string) ([]*types.ForeignKey, error) { + query := `select + kcu.COLUMN_NAME, kcu.CONSTRAINT_NAME, kcu.REFERENCED_TABLE_NAME, kcu.REFERENCED_COLUMN_NAME + from information_schema.KEY_COLUMN_USAGE kcu + inner join information_schema.TABLE_CONSTRAINTS tc + on tc.CONSTRAINT_NAME = kcu.CONSTRAINT_NAME + where tc.CONSTRAINT_TYPE = 'FOREIGN KEY' + and kcu.TABLE_NAME = ? + and kcu.CONSTRAINT_SCHEMA = ?` + + rows, err := m.db.Query(query, tableName, databaseName) + if err != nil { + return nil, err + } + + foreignKeys := make([]*types.ForeignKey, 0, 0) + for rows.Next() { + var childColumn, parentColumn, parentTable, name string + + if err := rows.Scan(&childColumn, &name, &parentTable, &parentColumn); err != nil { + return nil, err + } + + foreignKey := types.ForeignKey{ + Name: name, + ParentTable: parentTable, + ChildColumns: []string{childColumn}, + ParentColumns: []string{parentColumn}, + } + + for _, foundFk := range foreignKeys { + if foundFk.Name == name { + foundFk.ChildColumns = append(foreignKey.ChildColumns, childColumn) + foundFk.ParentColumns = append(foreignKey.ParentColumns, parentColumn) + + goto Appended + } + } + + foreignKeys = append(foreignKeys, &foreignKey) + + Appended: + } + + return foreignKeys, nil +} + +func (m *MysqlConnection) GetTablePrimaryKey(tableName string) ([]string, error) { + query := `select distinct(c.COLUMN_NAME) +from information_schema.TABLE_CONSTRAINTS tc +join information_schema.KEY_COLUMN_USAGE as kcu using (CONSTRAINT_SCHEMA, CONSTRAINT_NAME) +join information_schema.COLUMNS as c on c.TABLE_SCHEMA = tc.CONSTRAINT_SCHEMA + and tc.TABLE_NAME = c.TABLE_NAME + and kcu.TABLE_NAME = c.TABLE_NAME + and kcu.COLUMN_NAME = c.COLUMN_NAME +where tc.CONSTRAINT_TYPE = 'PRIMARY KEY' and tc.TABLE_NAME = ?` + + rows, err := m.db.Query(query, tableName) + if err != nil { + return nil, err + } + + columns := make([]string, 0, 0) + for rows.Next() { + var columnName string + + if err := rows.Scan(&columnName); err != nil { + return nil, err + } + + columns = append(columns, columnName) + } + + return columns, nil +} + +func (m *MysqlConnection) GetTableSchema(tableName string) ([]*types.Column, error) { + query := `select COLUMN_NAME, COLUMN_DEFAULT, IS_NULLABLE, DATA_TYPE, CHARACTER_MAXIMUM_LENGTH from information_schema.COLUMNS where TABLE_NAME = ? order by ORDINAL_POSITION` + rows, err := m.db.Query(query, tableName) + if err != nil { + return nil, err + } + + columns := make([]*types.Column, 0, 0) + + for rows.Next() { + column := types.Column{} + + var maxLength sql.NullInt64 + var isNullable string + var columnDefault sql.NullString + + if err := rows.Scan(&column.Name, &columnDefault, &isNullable, &column.DataType, &maxLength); err != nil { + return nil, err + } + + if isNullable == "NO" { + column.Constraints = &types.ColumnConstraints{ + NotNull: &trueValue, + } + } else { + column.Constraints = &types.ColumnConstraints{ + NotNull: &falseValue, + } + } + + if columnDefault.Valid { + column.ColumnDefault = &columnDefault.String + } + + if maxLength.Valid { + column.DataType = fmt.Sprintf("%s (%d)", column.DataType, maxLength.Int64) + } + columns = append(columns, &column) + } + + return columns, nil +} diff --git a/pkg/database/postgres/alter.go b/pkg/database/postgres/alter.go index dcc067c21..3a0bb28ef 100644 --- a/pkg/database/postgres/alter.go +++ b/pkg/database/postgres/alter.go @@ -7,9 +7,10 @@ import ( "github.com/lib/pq" schemasv1alpha1 "github.com/schemahero/schemahero/pkg/apis/schemas/v1alpha1" + "github.com/schemahero/schemahero/pkg/database/types" ) -func columnsMatch(col1 *Column, col2 *Column) bool { +func columnsMatch(col1 *types.Column, col2 *types.Column) bool { if col1.DataType != col2.DataType { return false } @@ -31,12 +32,12 @@ func columnsMatch(col1 *Column, col2 *Column) bool { return true } -func AlterColumnStatement(tableName string, desiredColumns []*schemasv1alpha1.SQLTableColumn, existingColumn *Column) (string, error) { +func AlterColumnStatement(tableName string, desiredColumns []*schemasv1alpha1.SQLTableColumn, existingColumn *types.Column) (string, error) { // this could be an alter or a drop column command columnStatement := "" for _, desiredColumn := range desiredColumns { if desiredColumn.Name == existingColumn.Name { - column, err := schemaColumnToPostgresColumn(desiredColumn) + column, err := schemaColumnToColumn(desiredColumn) if err != nil { return "", err } diff --git a/pkg/database/postgres/alter_test.go b/pkg/database/postgres/alter_test.go index 2a351337e..ddabdc848 100644 --- a/pkg/database/postgres/alter_test.go +++ b/pkg/database/postgres/alter_test.go @@ -4,6 +4,7 @@ import ( "testing" schemasv1alpha1 "github.com/schemahero/schemahero/pkg/apis/schemas/v1alpha1" + "github.com/schemahero/schemahero/pkg/database/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -14,7 +15,7 @@ func Test_AlterColumnStatment(t *testing.T) { name string tableName string desiredColumns []*schemasv1alpha1.SQLTableColumn - existingColumn *Column + existingColumn *types.Column expectedStatement string }{ { @@ -30,7 +31,7 @@ func Test_AlterColumnStatment(t *testing.T) { Type: "integer", }, }, - existingColumn: &Column{ + existingColumn: &types.Column{ Name: "b", DataType: "integer", ColumnDefault: nil, @@ -50,7 +51,7 @@ func Test_AlterColumnStatment(t *testing.T) { Type: "integer", }, }, - existingColumn: &Column{ + existingColumn: &types.Column{ Name: "b", DataType: "varchar(255)", ColumnDefault: nil, @@ -66,7 +67,7 @@ func Test_AlterColumnStatment(t *testing.T) { Type: "integer", }, }, - existingColumn: &Column{ + existingColumn: &types.Column{ Name: "b", DataType: "varchar(255)", ColumnDefault: nil, @@ -85,11 +86,11 @@ func Test_AlterColumnStatment(t *testing.T) { }, }, }, - existingColumn: &Column{ + existingColumn: &types.Column{ Name: "a", DataType: "integer", ColumnDefault: nil, - Constraints: &ColumnConstraints{ + Constraints: &types.ColumnConstraints{ NotNull: &falseValue, }, }, @@ -107,11 +108,11 @@ func Test_AlterColumnStatment(t *testing.T) { }, }, }, - existingColumn: &Column{ + existingColumn: &types.Column{ Name: "a", DataType: "integer", ColumnDefault: nil, - Constraints: &ColumnConstraints{ + Constraints: &types.ColumnConstraints{ NotNull: &trueValue, }, }, @@ -126,11 +127,11 @@ func Test_AlterColumnStatment(t *testing.T) { Type: "text", }, }, - existingColumn: &Column{ + existingColumn: &types.Column{ Name: "t", DataType: "text", ColumnDefault: nil, - Constraints: &ColumnConstraints{ + Constraints: &types.ColumnConstraints{ NotNull: &falseValue, }, }, @@ -148,11 +149,11 @@ func Test_AlterColumnStatment(t *testing.T) { }, }, }, - existingColumn: &Column{ + existingColumn: &types.Column{ Name: "ts", DataType: "timestamp", ColumnDefault: nil, - Constraints: &ColumnConstraints{ + Constraints: &types.ColumnConstraints{ NotNull: &trueValue, }, }, @@ -170,11 +171,11 @@ func Test_AlterColumnStatment(t *testing.T) { }, }, }, - existingColumn: &Column{ + existingColumn: &types.Column{ Name: "ts", DataType: "timestamp", ColumnDefault: nil, - Constraints: &ColumnConstraints{ + Constraints: &types.ColumnConstraints{ NotNull: &trueValue, }, }, diff --git a/pkg/database/postgres/column.go b/pkg/database/postgres/column.go index f06dfa302..179d18e9e 100644 --- a/pkg/database/postgres/column.go +++ b/pkg/database/postgres/column.go @@ -6,42 +6,14 @@ import ( "github.com/lib/pq" schemasv1alpha1 "github.com/schemahero/schemahero/pkg/apis/schemas/v1alpha1" + "github.com/schemahero/schemahero/pkg/database/types" ) -type ColumnConstraints struct { - NotNull *bool -} - -type Column struct { - Name string - DataType string - ColumnDefault *string - Constraints *ColumnConstraints -} - -func PostgresColumnToSchemaColumn(column *Column) (*schemasv1alpha1.SQLTableColumn, error) { - constraints := &schemasv1alpha1.SQLTableColumnConstraints{ - NotNull: column.Constraints.NotNull, - } - - schemaColumn := &schemasv1alpha1.SQLTableColumn{ - Name: column.Name, - Type: column.DataType, - Constraints: constraints, - } - - if column.ColumnDefault != nil { - schemaColumn.Default = *column.ColumnDefault - } - - return schemaColumn, nil -} - -func schemaColumnToPostgresColumn(schemaColumn *schemasv1alpha1.SQLTableColumn) (*Column, error) { - column := &Column{} +func schemaColumnToColumn(schemaColumn *schemasv1alpha1.SQLTableColumn) (*types.Column, error) { + column := &types.Column{} if schemaColumn.Constraints != nil { - column.Constraints = &ColumnConstraints{ + column.Constraints = &types.ColumnConstraints{ NotNull: schemaColumn.Constraints.NotNull, } } @@ -82,7 +54,7 @@ func postgresColumnAsInsert(column *schemasv1alpha1.SQLTableColumn) (string, err // 2. create table "users" ("id" bigint,"login" varchar(255),"name" varchar(255)) // if the column type is a known (safe) type, pass it unquoted, else pass whatever we received as quoted - postgresColumn, err := schemaColumnToPostgresColumn(column) + postgresColumn, err := schemaColumnToColumn(column) if err != nil { return "", err } diff --git a/pkg/database/postgres/column_test.go b/pkg/database/postgres/column_test.go index 03a5604a8..c483ad10b 100644 --- a/pkg/database/postgres/column_test.go +++ b/pkg/database/postgres/column_test.go @@ -4,6 +4,7 @@ import ( "testing" schemasv1alpha1 "github.com/schemahero/schemahero/pkg/apis/schemas/v1alpha1" + "github.com/schemahero/schemahero/pkg/database/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -117,7 +118,7 @@ func Test_schemaColumnToPostgresColumn(t *testing.T) { tests := []struct { name string schemaColumn *schemasv1alpha1.SQLTableColumn - expectedColumn *Column + expectedColumn *types.Column }{ { name: "text", @@ -125,7 +126,7 @@ func Test_schemaColumnToPostgresColumn(t *testing.T) { Name: "t", Type: "text", }, - expectedColumn: &Column{ + expectedColumn: &types.Column{ DataType: "text", ColumnDefault: nil, Constraints: nil, @@ -137,7 +138,7 @@ func Test_schemaColumnToPostgresColumn(t *testing.T) { Name: "c", Type: "character varying (10)", }, - expectedColumn: &Column{ + expectedColumn: &types.Column{ DataType: "character varying (10)", ColumnDefault: nil, }, @@ -148,7 +149,7 @@ func Test_schemaColumnToPostgresColumn(t *testing.T) { Name: "vc", Type: "varchar (10)", }, - expectedColumn: &Column{ + expectedColumn: &types.Column{ DataType: "character varying (10)", ColumnDefault: nil, }, @@ -159,7 +160,7 @@ func Test_schemaColumnToPostgresColumn(t *testing.T) { Name: "ip", Type: "cidr", }, - expectedColumn: &Column{ + expectedColumn: &types.Column{ DataType: "cidr", ColumnDefault: nil, Constraints: nil, @@ -171,7 +172,7 @@ func Test_schemaColumnToPostgresColumn(t *testing.T) { t.Run(test.name, func(t *testing.T) { req := require.New(t) - column, err := schemaColumnToPostgresColumn(test.schemaColumn) + column, err := schemaColumnToColumn(test.schemaColumn) req.NoError(err) assert.Equal(t, test.expectedColumn, column) }) diff --git a/pkg/database/postgres/create.go b/pkg/database/postgres/create.go index 3fa5210b4..350b29930 100644 --- a/pkg/database/postgres/create.go +++ b/pkg/database/postgres/create.go @@ -7,6 +7,7 @@ import ( "github.com/lib/pq" schemasv1alpha1 "github.com/schemahero/schemahero/pkg/apis/schemas/v1alpha1" + "github.com/schemahero/schemahero/pkg/database/types" ) func CreateTableStatement(tableName string, tableSchema *schemasv1alpha1.SQLTableSchema) (string, error) { @@ -28,6 +29,16 @@ func CreateTableStatement(tableName string, tableSchema *schemasv1alpha1.SQLTabl columns = append(columns, fmt.Sprintf("primary key (%s)", strings.Join(primaryKeyColumns, ", "))) } + if tableSchema.ForeignKeys != nil { + for _, foreignKey := range tableSchema.ForeignKeys { + columns = append(columns, fmt.Sprintf("constraint %q foreign key (%s) references %s (%s)", + types.GenerateFKName(tableName, foreignKey), + strings.Join(foreignKey.Columns, ", "), + foreignKey.References.Table, + strings.Join(foreignKey.References.Columns, ", "))) + } + } + query := fmt.Sprintf(`create table %s (%s)`, pq.QuoteIdentifier(tableName), strings.Join(columns, ", ")) return query, nil diff --git a/pkg/database/postgres/deploy.go b/pkg/database/postgres/deploy.go index 320a72c18..b6aa54479 100644 --- a/pkg/database/postgres/deploy.go +++ b/pkg/database/postgres/deploy.go @@ -6,19 +6,20 @@ import ( "fmt" schemasv1alpha1 "github.com/schemahero/schemahero/pkg/apis/schemas/v1alpha1" + "github.com/schemahero/schemahero/pkg/database/types" ) func DeployPostgresTable(uri string, tableName string, postgresTableSchema *schemasv1alpha1.SQLTableSchema) error { - db, err := sql.Open("postgres", uri) + p, err := Connect(uri) if err != nil { return err } - defer db.Close() + defer p.db.Close() // determine if the table exists query := `select count(1) from information_schema.tables where table_name = $1` fmt.Printf("Executing query %q\n", query) - row := db.QueryRow(query, tableName) + row := p.db.QueryRow(query, tableName) tableExists := 0 if err := row.Scan(&tableExists); err != nil { return err @@ -32,7 +33,7 @@ func DeployPostgresTable(uri string, tableName string, postgresTableSchema *sche } fmt.Printf("Executing query %q\n", query) - _, err = db.Exec(query) + _, err = p.db.Exec(query) if err != nil { return err } @@ -42,12 +43,11 @@ func DeployPostgresTable(uri string, tableName string, postgresTableSchema *sche // table needs to be altered? query = `select - column_name, column_default, is_nullable, data_type, - character_maximum_length + column_name, column_default, is_nullable, data_type, character_maximum_length from information_schema.columns where table_name = $1` fmt.Printf("Executing query %q\n", query) - rows, err := db.Query(query, tableName) + rows, err := p.db.Query(query, tableName) if err != nil { return err } @@ -58,17 +58,16 @@ func DeployPostgresTable(uri string, tableName string, postgresTableSchema *sche var columnDefault sql.NullString var charMaxLength sql.NullInt64 - if err := rows.Scan(&columnName, &columnDefault, &isNullable, &dataType, - &charMaxLength); err != nil { + if err := rows.Scan(&columnName, &columnDefault, &isNullable, &dataType, &charMaxLength); err != nil { return err } foundColumnNames = append(foundColumnNames, columnName) - existingColumn := Column{ + existingColumn := types.Column{ Name: columnName, DataType: dataType, - Constraints: &ColumnConstraints{}, + Constraints: &types.ColumnConstraints{}, } if isNullable == "NO" { @@ -110,9 +109,73 @@ func DeployPostgresTable(uri string, tableName string, postgresTableSchema *sche } } + // foreign key changes + droppedKeys := []string{} + currentForeignKeys, err := p.ListTableForeignKeys("", tableName) + if err != nil { + return err + } + for _, foreignKey := range postgresTableSchema.ForeignKeys { + var statement string + var err error + + var matchedForeignKey *types.ForeignKey + for _, currentForeignKey := range currentForeignKeys { + if currentForeignKey.Equals(types.SchemaForeignKeyToForeignKey(foreignKey)) { + goto Next + } + + matchedForeignKey = currentForeignKey + } + + // drop and readd? is this always ok + // TODO can we alter + if matchedForeignKey != nil { + statement, err = RemoveForeignKeyStatement(tableName, matchedForeignKey) + if err != nil { + return err + } + droppedKeys = append(droppedKeys, matchedForeignKey.Name) + alterAndDropStatements = append(alterAndDropStatements, statement) + } + + statement, err = AddForeignKeyStatement(tableName, foreignKey) + if err != nil { + return err + } + alterAndDropStatements = append(alterAndDropStatements, statement) + + Next: + } + + for _, currentForeignKey := range currentForeignKeys { + var statement string + var err error + + for _, foreignKey := range postgresTableSchema.ForeignKeys { + if currentForeignKey.Equals(types.SchemaForeignKeyToForeignKey(foreignKey)) { + goto NextCurrentFK + } + } + + for _, droppedKey := range droppedKeys { + if droppedKey == currentForeignKey.Name { + goto NextCurrentFK + } + } + + statement, err = RemoveForeignKeyStatement(tableName, currentForeignKey) + if err != nil { + return err + } + alterAndDropStatements = append(alterAndDropStatements, statement) + + NextCurrentFK: + } + for _, alterOrDropStatement := range alterAndDropStatements { fmt.Printf("Executing query %q\n", alterOrDropStatement) - if _, err = db.ExecContext(context.Background(), alterOrDropStatement); err != nil { + if _, err = p.db.ExecContext(context.Background(), alterOrDropStatement); err != nil { return err } } diff --git a/pkg/database/postgres/foreignkey.go b/pkg/database/postgres/foreignkey.go new file mode 100644 index 000000000..6dc330932 --- /dev/null +++ b/pkg/database/postgres/foreignkey.go @@ -0,0 +1,23 @@ +package postgres + +import ( + "fmt" + "strings" + + schemasv1alpha1 "github.com/schemahero/schemahero/pkg/apis/schemas/v1alpha1" + "github.com/schemahero/schemahero/pkg/database/types" +) + +func RemoveForeignKeyStatement(tableName string, foreignKey *types.ForeignKey) (string, error) { + return fmt.Sprintf("alter table %s drop constraint %s", tableName, foreignKey.Name), nil +} + +func AddForeignKeyStatement(tableName string, schemaForeignKey *schemasv1alpha1.SQLTableForeignKey) (string, error) { + return fmt.Sprintf("alter table %s add constraint %s foreign key (%s) references %s (%s)", + tableName, + types.GenerateFKName(tableName, schemaForeignKey), + strings.Join(schemaForeignKey.Columns, ", "), + schemaForeignKey.References.Table, + strings.Join(schemaForeignKey.References.Columns, ", ")), + nil +} diff --git a/pkg/database/postgres/tables.go b/pkg/database/postgres/tables.go index b87cd8df7..7c3335816 100644 --- a/pkg/database/postgres/tables.go +++ b/pkg/database/postgres/tables.go @@ -2,8 +2,10 @@ package postgres import ( "database/sql" + "fmt" _ "github.com/lib/pq" + "github.com/schemahero/schemahero/pkg/database/types" ) var ( @@ -32,7 +34,76 @@ func (p *PostgresConnection) ListTables() ([]string, error) { return tableNames, nil } +func (p *PostgresConnection) ListTableForeignKeys(databaseName string, tableName string) ([]*types.ForeignKey, error) { + // Starting with a query here: https://stackoverflow.com/questions/1152260/postgres-sql-to-list-table-foreign-keys + // TODO SchemaHero implementation needs to include a schema (database) here + // this is pg specific because composite fks need to be handled and this might be the only way? + query := `select + att2.attname as "child_column", + cl.relname as "parent_table", + att.attname as "parent_column", + conname + from + (select + unnest(con1.conkey) as "parent", + unnest(con1.confkey) as "child", + con1.confrelid, + con1.conrelid, + con1.conname + from + pg_class cl + join pg_namespace ns on cl.relnamespace = ns.oid + join pg_constraint con1 on con1.conrelid = cl.oid + where + cl.relname = $1 + and con1.contype = 'f' + ) con + join pg_attribute att on + att.attrelid = con.confrelid and att.attnum = con.child + join pg_class cl on + cl.oid = con.confrelid + join pg_attribute att2 on + att2.attrelid = con.conrelid and att2.attnum = con.parent` + + rows, err := p.db.Query(query, tableName) + if err != nil { + return nil, err + } + + foreignKeys := make([]*types.ForeignKey, 0, 0) + for rows.Next() { + var childColumn, parentColumn, parentTable, name string + + if err := rows.Scan(&childColumn, &parentTable, &parentColumn, &name); err != nil { + return nil, err + } + + foreignKey := types.ForeignKey{ + Name: name, + ParentTable: parentTable, + ChildColumns: []string{childColumn}, + ParentColumns: []string{parentColumn}, + } + + for _, foundFk := range foreignKeys { + if foundFk.Name == name { + foundFk.ChildColumns = append(foreignKey.ChildColumns, childColumn) + foundFk.ParentColumns = append(foreignKey.ParentColumns, parentColumn) + + goto Appended + } + } + + foreignKeys = append(foreignKeys, &foreignKey) + + Appended: + } + + return foreignKeys, nil +} + func (p *PostgresConnection) GetTablePrimaryKey(tableName string) ([]string, error) { + // TODO we should be adding a database name on this select query := `select c.column_name from information_schema.table_constraints tc join information_schema.constraint_column_usage as ccu using (constraint_schema, constraint_name) @@ -59,7 +130,7 @@ where constraint_type = 'PRIMARY KEY' and tc.table_name = $1` return columns, nil } -func (p *PostgresConnection) GetTableSchema(tableName string) ([]*Column, error) { +func (p *PostgresConnection) GetTableSchema(tableName string) ([]*types.Column, error) { query := "select column_name, data_type, character_maximum_length, column_default, is_nullable from information_schema.columns where table_name = $1" rows, err := p.db.Query(query, tableName) @@ -67,9 +138,9 @@ func (p *PostgresConnection) GetTableSchema(tableName string) ([]*Column, error) return nil, err } - columns := make([]*Column, 0, 0) + columns := make([]*types.Column, 0, 0) for rows.Next() { - column := Column{} + column := types.Column{} var maxLength sql.NullInt64 var isNullable string @@ -80,11 +151,11 @@ func (p *PostgresConnection) GetTableSchema(tableName string) ([]*Column, error) } if isNullable == "NO" { - column.Constraints = &ColumnConstraints{ + column.Constraints = &types.ColumnConstraints{ NotNull: &trueValue, } } else { - column.Constraints = &ColumnConstraints{ + column.Constraints = &types.ColumnConstraints{ NotNull: &falseValue, } } @@ -93,6 +164,10 @@ func (p *PostgresConnection) GetTableSchema(tableName string) ([]*Column, error) column.ColumnDefault = &columnDefault.String } + if maxLength.Valid { + column.DataType = fmt.Sprintf("%s (%d)", column.DataType, maxLength.Int64) + } + columns = append(columns, &column) } diff --git a/pkg/database/types/column.go b/pkg/database/types/column.go new file mode 100644 index 000000000..43f7c32f8 --- /dev/null +++ b/pkg/database/types/column.go @@ -0,0 +1,35 @@ +package types + +import ( + schemasv1alpha1 "github.com/schemahero/schemahero/pkg/apis/schemas/v1alpha1" +) + +type ColumnConstraints struct { + NotNull *bool +} + +type Column struct { + Name string + DataType string + ColumnDefault *string + Constraints *ColumnConstraints +} + +func ColumnToSchemaColumn(column *Column) (*schemasv1alpha1.SQLTableColumn, error) { + schemaColumn := &schemasv1alpha1.SQLTableColumn{ + Name: column.Name, + Type: column.DataType, + } + + if column.Constraints != nil { + schemaColumn.Constraints = &schemasv1alpha1.SQLTableColumnConstraints{ + NotNull: column.Constraints.NotNull, + } + } + + if column.ColumnDefault != nil { + schemaColumn.Default = *column.ColumnDefault + } + + return schemaColumn, nil +} diff --git a/pkg/database/types/foreignkey.go b/pkg/database/types/foreignkey.go new file mode 100644 index 000000000..127837608 --- /dev/null +++ b/pkg/database/types/foreignkey.go @@ -0,0 +1,52 @@ +package types + +import ( + "fmt" + "strings" + + schemasv1alpha1 "github.com/schemahero/schemahero/pkg/apis/schemas/v1alpha1" +) + +type ForeignKey struct { + ChildColumns []string + ParentTable string + ParentColumns []string + Name string +} + +func (fk *ForeignKey) Equals(other *ForeignKey) bool { + // TODO + + return false +} + +func ForeignKeyToSchemaForeignKey(foreignKey *ForeignKey) *schemasv1alpha1.SQLTableForeignKey { + schemaForeignKey := schemasv1alpha1.SQLTableForeignKey{ + Columns: foreignKey.ChildColumns, + References: schemasv1alpha1.SQLTableForeignKeyReferences{ + Table: foreignKey.ParentTable, + Columns: foreignKey.ParentColumns, + }, + Name: foreignKey.Name, + } + + return &schemaForeignKey +} + +func SchemaForeignKeyToForeignKey(schemaForeignKey *schemasv1alpha1.SQLTableForeignKey) *ForeignKey { + foreignKey := ForeignKey{ + ChildColumns: schemaForeignKey.Columns, + ParentTable: schemaForeignKey.References.Table, + ParentColumns: schemaForeignKey.References.Columns, + } + + return &foreignKey +} + +func GenerateFKName(tableName string, schemaForeignKey *schemasv1alpha1.SQLTableForeignKey) string { + if schemaForeignKey.Name != "" { + return schemaForeignKey.Name + } + + return fmt.Sprintf("%s_%s_fkey", tableName, strings.Join(schemaForeignKey.Columns, "_")) +} diff --git a/pkg/generate/generate.go b/pkg/generate/generate.go index a2dee9922..9ff69fde6 100644 --- a/pkg/generate/generate.go +++ b/pkg/generate/generate.go @@ -7,7 +7,10 @@ import ( "strings" schemasv1alpha1 "github.com/schemahero/schemahero/pkg/apis/schemas/v1alpha1" + "github.com/schemahero/schemahero/pkg/database/interfaces" + "github.com/schemahero/schemahero/pkg/database/mysql" "github.com/schemahero/schemahero/pkg/database/postgres" + "github.com/schemahero/schemahero/pkg/database/types" "github.com/spf13/viper" "gopkg.in/yaml.v2" ) @@ -25,85 +28,63 @@ func NewGenerator() *Generator { func (g *Generator) RunSync() error { fmt.Printf("connecting to %s\n", g.Viper.GetString("uri")) - db, err := postgres.Connect(g.Viper.GetString("uri")) - if err != nil { - return err + var db interfaces.SchemaHeroDatabaseConnection + if g.Viper.GetString("driver") == "postgres" { + pgDb, err := postgres.Connect(g.Viper.GetString("uri")) + if err != nil { + return err + } + db = pgDb + } else if g.Viper.GetString("driver") == "mysql" { + mysqlDb, err := mysql.Connect(g.Viper.GetString("uri")) + if err != nil { + return err + } + db = mysqlDb } - tables, err := db.ListTables() + tableNames, err := db.ListTables() if err != nil { fmt.Printf("%#v\n", err) return err } filesWritten := make([]string, 0, 0) - for _, table := range tables { - primaryKey, err := db.GetTablePrimaryKey(table) + for _, tableName := range tableNames { + primaryKey, err := db.GetTablePrimaryKey(tableName) if err != nil { fmt.Printf("%#v\n", err) return err } - columns, err := db.GetTableSchema(table) + foreignKeys, err := db.ListTableForeignKeys(g.Viper.GetString("dbname"), tableName) if err != nil { fmt.Printf("%#v\n", err) return err } - postgresTableColumns := make([]*schemasv1alpha1.SQLTableColumn, 0, 0) - - for _, column := range columns { - postgresTableColumn, err := postgres.PostgresColumnToSchemaColumn(column) - if err != nil { - fmt.Printf("%#v\n", err) - return err - } - - postgresTableColumns = append(postgresTableColumns, postgresTableColumn) - } - - postgresTableSchema := schemasv1alpha1.SQLTableSchema{ - PrimaryKey: primaryKey, - Columns: postgresTableColumns, - } - - schemaHeroResource := schemasv1alpha1.TableSpec{ - Database: g.Viper.GetString("dbname"), - Name: table, - Requires: []string{}, - Schema: &schemasv1alpha1.TableSchema{ - Postgres: &postgresTableSchema, - }, - } - - specDoc := struct { - Spec schemasv1alpha1.TableSpec `yaml:"spec"` - }{ - schemaHeroResource, + columns, err := db.GetTableSchema(tableName) + if err != nil { + fmt.Printf("%#v\n", err) + return err } - b, err := yaml.Marshal(&specDoc) + tableYAML, err := generateTableYAML(g.Viper.GetString("driver"), g.Viper.GetString("dbname"), tableName, primaryKey, foreignKeys, columns) if err != nil { fmt.Printf("%#v\n", err) return err } - // TODO consider marshaling this instead of inline - tableDoc := fmt.Sprintf(`apiVersion: schemas.schemahero.io/v1alpha1 -kind: Table -metadata: - name: %s -%s`, sanitizeName(table), b) - // If there was a outputdir set, write it, else print it if g.Viper.GetString("output-dir") != "" { - if err := ioutil.WriteFile(filepath.Join(g.Viper.GetString("output-dir"), fmt.Sprintf("%s.yaml", table)), []byte(tableDoc), 0644); err != nil { + if err := ioutil.WriteFile(filepath.Join(g.Viper.GetString("output-dir"), fmt.Sprintf("%s.yaml", sanitizeName(tableName))), []byte(tableYAML), 0644); err != nil { return err } - filesWritten = append(filesWritten, fmt.Sprintf("./%s.yaml", table)) + filesWritten = append(filesWritten, fmt.Sprintf("./%s.yaml", sanitizeName(tableName))) } else { - fmt.Println(tableDoc) + + fmt.Println(tableYAML) fmt.Println("---") } } @@ -128,6 +109,68 @@ metadata: return nil } +func generateTableYAML(driver string, dbName string, tableName string, primaryKey []string, foreignKeys []*types.ForeignKey, columns []*types.Column) (string, error) { + schemaForeignKeys := make([]*schemasv1alpha1.SQLTableForeignKey, 0, 0) + for _, foreignKey := range foreignKeys { + schemaForeignKey := types.ForeignKeyToSchemaForeignKey(foreignKey) + schemaForeignKeys = append(schemaForeignKeys, schemaForeignKey) + } + + schemaTableColumns := make([]*schemasv1alpha1.SQLTableColumn, 0, 0) + for _, column := range columns { + schemaTableColumn, err := types.ColumnToSchemaColumn(column) + if err != nil { + fmt.Printf("%#v\n", err) + return "", err + } + + schemaTableColumns = append(schemaTableColumns, schemaTableColumn) + } + + tableSchema := &schemasv1alpha1.SQLTableSchema{ + PrimaryKey: primaryKey, + Columns: schemaTableColumns, + ForeignKeys: schemaForeignKeys, + } + + schema := &schemasv1alpha1.TableSchema{} + + if driver == "postgres" { + schema.Postgres = tableSchema + } else if driver == "mysql" { + schema.Mysql = tableSchema + } + + schemaHeroResource := schemasv1alpha1.TableSpec{ + Database: dbName, + Name: tableName, + Requires: []string{}, + Schema: schema, + } + + specDoc := struct { + Spec schemasv1alpha1.TableSpec `yaml:"spec"` + }{ + schemaHeroResource, + } + + b, err := yaml.Marshal(&specDoc) + if err != nil { + fmt.Printf("%#v\n", err) + return "", err + } + + // TODO consider marshaling this instead of inline + tableDoc := fmt.Sprintf(`apiVersion: schemas.schemahero.io/v1alpha1 +kind: Table +metadata: + name: %s +%s`, sanitizeName(tableName), b) + + return tableDoc, nil + +} + func sanitizeName(name string) string { return strings.Replace(name, "_", "-", -1) } diff --git a/pkg/generate/generate_test.go b/pkg/generate/generate_test.go index bfa08c32d..fa02f5f57 100644 --- a/pkg/generate/generate_test.go +++ b/pkg/generate/generate_test.go @@ -3,7 +3,9 @@ package generate import ( "testing" + "github.com/schemahero/schemahero/pkg/database/types" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func Test_sanitizeName(t *testing.T) { @@ -18,3 +20,106 @@ func Test_sanitizeName(t *testing.T) { }) } } + +func Test_writeTableFile(t *testing.T) { + tests := []struct { + name string + driver string + dbName string + tableName string + primaryKey []string + foreignKeys []*types.ForeignKey + columns []*types.Column + expectedYAML string + }{ + { + name: "pg 1 col", + driver: "postgres", + dbName: "db", + tableName: "simple", + primaryKey: []string{"one"}, + foreignKeys: []*types.ForeignKey{}, + columns: []*types.Column{ + &types.Column{ + Name: "id", + DataType: "integer", + }, + }, + expectedYAML: `apiVersion: schemas.schemahero.io/v1alpha1 +kind: Table +metadata: + name: simple +spec: + database: db + name: simple + schema: + postgres: + primaryKey: + - one + columns: + - name: id + type: integer +`, + }, + { + name: "pg foreign key", + driver: "postgres", + dbName: "db", + tableName: "withfk", + primaryKey: []string{"pk"}, + foreignKeys: []*types.ForeignKey{ + &types.ForeignKey{ + ChildColumns: []string{"cc"}, + ParentTable: "p", + ParentColumns: []string{"pc"}, + Name: "fk_pc_cc", + }, + }, + columns: []*types.Column{ + &types.Column{ + Name: "pk", + DataType: "integer", + }, + &types.Column{ + Name: "cc", + DataType: "integer", + }, + }, + expectedYAML: `apiVersion: schemas.schemahero.io/v1alpha1 +kind: Table +metadata: + name: withfk +spec: + database: db + name: withfk + schema: + postgres: + primaryKey: + - pk + foreignKeys: + - columns: + - cc + references: + table: p + columns: + - pc + name: fk_pc_cc + columns: + - name: pk + type: integer + - name: cc + type: integer +`, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + req := require.New(t) + + y, err := generateTableYAML(test.driver, test.dbName, test.tableName, test.primaryKey, test.foreignKeys, test.columns) + req.NoError(err) + assert.Equal(t, test.expectedYAML, y) + }) + } +}