Foreign key support

.buildkite/pipeline.yml

@@ -14,30 +14,14 @@ steps:
 
   - wait
 
-  - commands:
-      - make -C integration integration-test-image TAG="${BUILDKITE_COMMIT:0:7}"
+  - label: integration-tests
+    commands:
+      - make -C integration run
 
   - wait
 
-  # - commands:
-  #     docker run -it --rm \
-  #       --volume /var/lib/buildkite-agent/builds/buildkite-wdxz-1/replicated/schemahero:/go/src/github.com/schemahero/schemahero \
-  #       --volume /var/run/docker.sock:/var/run/docker.sock \
-  #       --workdir /go/src/github.com/schemahero/schemahero \
-  #       --env BUILDKITE_JOB_ID \
-  #       --env BUILDKITE_BUILD_ID \
-  #       --env BUILDKITE_AGENT_ACCESS_TOKEN \
-  #       --userns=host \
-  #       --volume /usr/bin/buildkite-agent:/usr/bin/buildkite-agent \
-  #       replicated/gitops-builder:buildkite-go12-node10 \
-  #       /bin/sh -e -c \
-  #       make -C integration run
-
-  # - wait
-
   - commands:
       - bash <(curl -s https://codecov.io/bash)
-    branches: "master"
 
   - commands:
       - make snapshot-release

README.md

@@ -6,16 +6,13 @@
 [![Godoc](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)](https://godoc.org/github.com/schemahero/schemahero)
 [![LICENSE](https://img.shields.io/github/license/schemahero/schemahero.svg?style=flat-square)](https://github.com/schemahero/schemahero/blob/master/LICENSE)
 
-**Note**: This is a work-in-progress that is not yet functional. SchemaHero is a an experiment right now, and does not have enough implementation to be used in any environment. Work is in progress.
-
 ## What is SchemaHero?
 
-SchemaHero is a Kubernetes Operator for Declarative Schema Management for various databases. SchemaHero has the following goals:
+SchemaHero is a Kubernetes Operator for [Declarative Schema Management](https://schemahero.io/background/declarative-schema-management/) for [various databases](https://schemahero.io/databases/). SchemaHero has the following goals:
 
-1. Database tables can be expressed as [Kubernetes resources](https://github.com/schemahero/schemahero/blob/master/config/samples/schemas_v1alpha1_table.yaml) that can be updated and deployed to the cluster.
-2. Database migrations can be written as SQL statements, expressed as [Kubernetes resources](https://github.com/schemahero/schemahero/blob/master/config/samples/schemas_v1alpha1_migration.yaml) that can be deployed to the cluster.
-3. Database schemas can be [monitored for drift](https://github.com/schemahero/schemahero/blob/master/config/samples/databases_v1alpha1_database.yaml) and brought back to the desired state automatically.
-4. Schemas and migations can [require other schemas or migrations](https://github.com/schemahero/schemahero/blob/master/config/samples/schemas_v1alpha1_table.yaml#L30) instead of ordering with timestamps and/or sequences.
+1. Database table schemas can be expressed as [Kubernetes resources](https://schemahero.io/how-to-use/deploying-tables/creating-tables/) that can be deployed to a cluster.
+2. Database schemas can be edited and deployed to the cluster. SchemaHero will calculate the required change (`ALTER TABLE` statement) and apply it.
+3. SchemaHero can manage databases that are deployed to the cluster, or external to the cluster (RDS, Google CloudSQL, etc).
 
 ## Getting Started
 

config/crds/schemas_v1alpha1_table.yaml

@@ -59,6 +59,32 @@ spec:
                         - type
                         type: object
                       type: array
+                    foreignKeys:
+                      items:
+                        properties:
+                          columns:
+                            items:
+                              type: string
+                            type: array
+                          name:
+                            type: string
+                          references:
+                            properties:
+                              columns:
+                                items:
+                                  type: string
+                                type: array
+                              table:
+                                type: string
+                            required:
+                            - table
+                            - columns
+                            type: object
+                        required:
+                        - columns
+                        - references
+                        type: object
+                      type: array
                     isDeleted:
                       type: boolean
                     primaryKey:
@@ -89,6 +115,32 @@ spec:
                         - type
                         type: object
                       type: array
+                    foreignKeys:
+                      items:
+                        properties:
+                          columns:
+                            items:
+                              type: string
+                            type: array
+                          name:
+                            type: string
+                          references:
+                            properties:
+                              columns:
+                                items:
+                                  type: string
+                                type: array
+                              table:
+                                type: string
+                            required:
+                            - table
+                            - columns
+                            type: object
+                        required:
+                        - columns
+                        - references
+                        type: object
+                      type: array
                     isDeleted:
                       type: boolean
                     primaryKey:
@@ -102,7 +154,6 @@ spec:
           required:
           - database
           - name
-          - requires
           - schema
           type: object
         status:

config/dev/database/postgres-10.7.yaml

@@ -17,10 +17,11 @@ spec:
   ports:
   - name: postgresql
     port: 5432
+    nodePort: 30432
     targetPort: postgresql
   selector:
     app: postgresql
-  type: ClusterIP
+  type: NodePort
 ---
 apiVersion: apps/v1beta2
 kind: StatefulSet

config/dev/github/projects-table-pg.yaml

@@ -0,0 +1,23 @@
+apiVersion: schemas.schemahero.io/v1alpha1
+kind: Table
+metadata:
+  name: projects
+spec:
+  database: testdb
+  name: projects
+  requires: []
+  schema:
+    postgres:
+      primaryKey: [id]
+      foreignKeys:
+        - columns:
+            - id
+          references:
+            table: users
+            columns:
+              - id
+      columns:
+        - name: id
+          type: integer
+        - name: user_id
+          type: integer

config/dev/github/projects.yaml

@@ -0,0 +1,20 @@
+database: testdb
+name: projects
+requires: []
+schema:
+  postgres:
+    primaryKey: [id]
+    foreignKeys:
+      - columns:
+          - id
+        references:
+          table: users
+          columns:
+            - id
+    columns:
+      - name: id
+        type: integer
+      - name: name
+        type: varchar(255)
+      - name: user_id
+        type: integer

config/dev/github/users-table-pg.yaml

@@ -1,8 +1,6 @@
 apiVersion: schemas.schemahero.io/v1alpha1
 kind: Table
 metadata:
-  labels:
-    controller-tools.k8s.io: "1.0"
   name: users
 spec:
   database: testdb

config/dev/github/users.yaml

@@ -0,0 +1,15 @@
+database: testdb
+name: users
+requires: []
+schema:
+  postgres:
+    primaryKey: [id]
+    columns:
+      - name: id
+        type: integer
+      - name: login
+        type: varchar(255)
+      - name: name
+        type: varchar(255)
+
+

integration/.gitignore

@@ -0,0 +1 @@
+out/

integration/Makefile

@@ -1,28 +1,22 @@
-export GO_BUILD=env GO111MODULE=on go build
 
-TAG ?= $(shell uuidgen)
+SHELL := /bin/bash
+IMAGE := "ttl.sh/$(shell uuidgen):1h"
+export IMAGE
 
 .PHONY: run
-run: build
-	bin/schemahero-integration-tests run \
-		--manager-image-name "ttl.sh/sh/manager-$(TAG):1h" \
-		--schemahero-image-name "ttl.sh/sh/$(TAG):1h"
+run: build postgres mysql
 
-integration-test-image:
-	docker build -t ttl.sh/sh/$(TAG):1h -f ../Dockerfile.schemahero ..
-	docker build -t ttl.sh/sh/manager-$(TAG):1h -f ../Dockerfile.manager ..
-	docker push ttl.sh/sh/$(TAG):1h
-	docker push ttl.sh/sh/manager-$(TAG):1h
+.PHONY: postgres
+postgres: build
+	# make -C tests/postgres/create-table run
+	# make -C tests/postgres/foreign-key run
+
+.PHONY: mysql
+mysql: build
+	# make -C tests/mysql/create-table run
+	make -C tests/mysql/foreign-key run
 
 .PHONY: build
-build: GO111MODULE = "on"
 build:
-	rm -rf bin/schemahero-integration-tests
-	$(GO_BUILD) \
-		-ldflags "\
-			-X ${VERSION_PACKAGE}.version=${VERSION} \
-			-X ${VERSION_PACKAGE}.gitSHA=${GIT_SHA} \
-			-X ${VERSION_PACKAGE}.buildTime=${DATE}" \
-		-o bin/schemahero-integration-tests \
-		./cmd/integration
-	@echo "built bin/schemahero-integration-tests"
+	docker build -t $(IMAGE) -f ../Dockerfile.schemahero ..
+	docker push $(IMAGE)

integration/README.md

@@ -1,2 +1,11 @@
 # Schemahero Integration Tests
 
+These tests verify SchemaHero by:
+
+1. Creating a database with an init script so that there are some predefined tables
+2. Applying a table.yaml
+3. Generating fixtures using schemahero
+4. Verifying that the generated fixtures are correct
+
+
+Ideally these tests should be executed from Go code for more reliability and easier to use.