Merge pull request #5 from schemahero/table-schema-test

Table schema test

.codecov.yml

@@ -0,0 +1,3 @@
+ignore:
+  - "**/zz_generated.deepcopy.go"  # Ignore generated deepcopy code from kubebuilder
+  - "pkg/client/schemaheroclientset/**/*.go" # Ignore generated code

README.md

@@ -10,7 +10,7 @@
 
 ## What is SchemaHero?
 
-SchemaHero is a Kubernetes Operator to enable Declarative Schema Management for databases.
+SchemaHero is a Kubernetes Operator for Declarative Schema Management for various databases
 
 1. Database tables can be expressed as [Kubernetes resources](https://github.com/schemahero/schemahero/blob/master/config/samples/schemas_v1alpha1_table.yaml) that can be updated and deployed to the cluster.
 2. Database migrations can be written as SQL statements, expressed as [Kubernetes resources](https://github.com/schemahero/schemahero/blob/master/config/samples/schemas_v1alpha1_migration.yaml) that can be deployed to the cluster.

config/crds/databases_v1alpha1_database.yaml

@@ -52,6 +52,11 @@ spec:
           type: string
         metadata:
           type: object
+        schemahero:
+          properties:
+            image:
+              type: string
+          type: object
         status:
           properties:
             isConnected:

config/dev/github/pg.yaml

@@ -4,6 +4,8 @@ metadata:
   labels:
     controller-tools.k8s.io: "1.0"
   name: github-pg
+schemahero:
+  image: "localhost:32000/schemahero/schemahero"
 connection:
   postgres:
     uri:

deploy/.goreleaser.integration.yml

@@ -29,18 +29,19 @@ builds:
     flags: -tags netgo -installsuffix netgo
     binary: manager
     hooks: {}
-archive:
-  format: tar.gz
-  name_template: '{{ .Binary }}_{{.Version}}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{.Arm }}{{ end }}-alpha'
-  files:
-    - licence*
-    - LICENCE*
-    - license*
-    - LICENSE*
-    - readme*
-    - README*
-    - changelog*
-    - CHANGELOG*
+archives:
+  - id: tar
+    format: tar.gz
+    name_template: '{{ .Binary }}_{{.Version}}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{.Arm }}{{ end }}-alpha'
+    files:
+      - licence*
+      - LICENCE*
+      - license*
+      - LICENSE*
+      - readme*
+      - README*
+      - changelog*
+      - CHANGELOG*
 dockers:
   - dockerfile: ./deploy/Dockerfile.schemahero
     image_templates:

deploy/.goreleaser.yml

@@ -32,18 +32,19 @@ builds:
     flags: -tags netgo -installsuffix netgo
     binary: manager
     hooks: {}
-archive:
-  format: tar.gz
-  name_template: '{{ .Binary }}_{{.Version}}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{.Arm }}{{ end }}-alpha'
-  files:
-    - licence*
-    - LICENCE*
-    - license*
-    - LICENSE*
-    - readme*
-    - README*
-    - changelog*
-    - CHANGELOG*
+archived:
+  - id: tar
+    format: tar.gz
+    name_template: '{{ .Binary }}_{{.Version}}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{.Arm }}{{ end }}-alpha'
+    files:
+      - licence*
+      - LICENCE*
+      - license*
+      - LICENSE*
+      - readme*
+      - README*
+      - changelog*
+      - CHANGELOG*
 dockers:
   - dockerfile: ./deploy/Dockerfile.schemahero
     image_templates:

docs/design/declarative-schema-management.md

@@ -0,0 +1,39 @@
+# Desired Schema Management
+
+There are many benefits to managing database schemas as code, including:
+1. Ability to adhere to a change management process
+2. Repeatable deployments to new environments
+...
+
+## Current Tools
+
+There are several commonly-used methods of managing database schemas employed today.
+
+1. Sequencial Migrations / Replay (Convergence)
+1. Declarative Schema Management
+
+### Sequential Migrations / Replay
+
+Tools such as db-migrate, Flyaway, goose and others fit into this category. These are immutable after deployment, stacked migrations that start from an empty database and consist of ordered create, alter and drop commands.
+
+#### Challenges
+
+These tools work nicely at first, but over time create some operational challenges:
+
+1. When a feature is no longer used, the database runtime must continue to support this to allow this migration to succeed. For example, if using Postgres and an extension is required and used, and then removed, the database migations will fail to run on a new database unless that extension is present, even though it's not needed in the end state.
+
+2. Performance starts to become slow on new environments. Eventually, in a rapid-iteration product, there can be hundreds of migrations. Replying these can be slow and any single failed migration will break the deployment.
+
+3. Database upgrades create incompatible migrations. After upgrading a database version, the syntax supported may change. This can leave older migrations unable to be applied against the current version of the databse.
+
+4. Concurrent changes can create conflicts or skipped migrations. These tools often employ a sequential (integer) counter or a timestamp. When multiple migrations are simultaneously prepared offline, these may have the same counter value or be commited in a different order than they were generated. This can cause the runtime to skip a migration.
+
+5. No dependency management between teams.
+
+#### Workarounds
+
+To help solve this, manual intervention is often taken to "rebase" the migrations. This is equivalent to retrieving a current schema from the database, deleting all migrations, and creating a single migration. This is a manual process that must be run occaisionally when using a sequential migration strategy.
+
+#### Benefits
+
+1. Ordering of columns is guaranteed.

docs/design/desired-state-database-schema-management.md

@@ -1,39 +1,3 @@
-# Desired State Database Schema Management
+This document has moved to:
 
-There are many benefits to managing database schemas as code, including:
-1. Ability to adhere to a change management process
-2. Repeatable deployments to new environments
-...
-
-## Current Tools
-
-There are several commonly-used methods of managing database schemas employed today.
-
-1. Sequencial Migrations / Replay (Convergence)
-1. Declarative Schema Management
-
-### Sequential Migrations / Replay
-
-Tools such as db-migrate, Flyaway, goose and others fit into this category. These are immutable after deployment, stacked migrations that start from an empty database and consist of ordered create, alter and drop commands.
-
-#### Challenges
-
-These tools work nicely at first, but over time create some operational challenges:
-
-1. When a feature is no longer used, the database runtime must continue to support this to allow this migration to succeed. For example, if using Postgres and an extension is required and used, and then removed, the database migations will fail to run on a new database unless that extension is present, even though it's not needed in the end state.
-
-2. Performance starts to become slow on new environments. Eventually, in a rapid-iteration product, there can be hundreds of migrations. Replying these can be slow and any single failed migration will break the deployment.
-
-3. Database upgrades create incompatible migrations. After upgrading a database version, the syntax supported may change. This can leave older migrations unable to be applied against the current version of the databse.
-
-4. Concurrent changes can create conflicts or skipped migrations. These tools often employ a sequential (integer) counter or a timestamp. When multiple migrations are simultaneously prepared offline, these may have the same counter value or be commited in a different order than they were generated. This can cause the runtime to skip a migration.
-
-5. No dependency management between teams.
-
-#### Workarounds
-
-To help solve this, manual intervention is often taken to "rebase" the migrations. This is equivalent to retrieving a current schema from the database, deleting all migrations, and creating a single migration. This is a manual process that must be run occaisionally when using a sequential migration strategy.
-
-#### Benefits
-
-1. Ordering of columns is guaranteed.
+[https://github.com/schemahero/schemahero/blob/master/docs/design/desired-state-database-schema-management.md](https://github.com/schemahero/schemahero/blob/master/docs/design/desired-state-database-schema-management.md).

docs/design/schema-migrations-and-data-migations.md

@@ -0,0 +1,16 @@
+# Schema Migrations and Data Migations
+
+There are two types of migratinos that have to be managed and deployed:
+
+1. Schema Migrations
+1. Data Migrations
+
+## Schema Migrations
+
+A Schema migration can be expressed in SQL syntax, and alters the structure of the database. These often are new tables, changing columns, altering indexed data and more. These are commonly written and can always be expressed in an idempotent syntax. Different database engines enforce various rules on how these can be applied. For example, MySQL will not allow a schema migration to be executed in a transaction, while Postgres will. Schema management is often unique to the database.
+
+## Data Migrations
+
+Less frequently, a developer must migrate some data to a new format in a database. This can involve calculating a new column and writing it, or creating new values in code and inserting them. Many traditional database management tools blend the tasks of schema migrations and data migrations into one tool.
+
+When looking at adding a data migration to a project, there is often a way to achieve the same result by implementing the update differently.

integration/Makefile

@@ -1,17 +1,19 @@
 export GO_BUILD=env GO111MODULE=on go build
 MANAGER_IMAGE_NAME := $(shell uuidgen)
-SCHEMA_HERO_IMAGE_NAME := $(shell uuidgen)
+SCHEMAHERO_IMAGE_NAME := $(shell uuidgen)
 
 .PHONY: run
 run: build integration-test-image
-	bin/schemahero-integration-tests run --manager-image-name "rgstry.dev/$(MANAGER_IMAGE_NAME):1h"
+	bin/schemahero-integration-tests run \
+		--manager-image-name "replreg.is/$(MANAGER_IMAGE_NAME):1h" \
+		--schemahero-image-name "replreg.is/$(SCHEMAHERO_IMAGE_NAME):1h"
 
 integration-test-image:
 	cd .. && curl -sL https://git.io/goreleaser | bash -s -- --snapshot --rm-dist --config deploy/.goreleaser.integration.yml
-	docker tag schemahero/schemahero:integration-test rgstry.dev/$(SCHEMA_HERO_IMAGE_NAME):1h
-	docker tag schemahero/schemahero-manager:integration-test rgstry.dev/$(MANAGER_IMAGE_NAME):1h
-	docker push rgstry.dev/$(SCHEMA_HERO_IMAGE_NAME):1h
-	docker push rgstry.dev/$(MANAGER_IMAGE_NAME):1h
+	docker tag schemahero/schemahero:integration-test replreg.is/$(SCHEMAHERO_IMAGE_NAME):1h
+	docker tag schemahero/schemahero-manager:integration-test replreg.is/$(MANAGER_IMAGE_NAME):1h
+	docker push replreg.is/$(SCHEMAHERO_IMAGE_NAME):1h
+	docker push replreg.is/$(MANAGER_IMAGE_NAME):1h
 
 .PHONY: build
 build: GO111MODULE = "on"

integration/pkg/cli/run.go

@@ -18,7 +18,8 @@ func Run() *cobra.Command {
 		},
 	}
 
-	cmd.Flags().String("manager-image-name", "", "docker image name for the manager to test with")
+	cmd.Flags().String("manager-image-name", "", "docker image for the manager pod")
+	cmd.Flags().String("schemahero-image-name", "", "docker image for the schemahero pod")
 
 	viper.BindPFlags(cmd.Flags())
 

integration/pkg/runner/cluster.go

@@ -80,29 +80,15 @@ func (c Cluster) delete() error {
 	return ctx.Delete()
 }
 
-func (c Cluster) apply(manifests []byte, showStdOut bool) error {
+func (c Cluster) kubectl(ctx context.Context, cmd []string) (*container.Config, *container.HostConfig, error) {
 	cli, err := client.NewEnvClient()
 	if err != nil {
-		return err
-	}
-
-	ctx := context.Background()
-
-	tmpFile, err := ioutil.TempFile(os.TempDir(), "manifests-")
-	if err != nil {
-		return err
-	}
-	defer os.Remove(tmpFile.Name())
-	if _, err = tmpFile.Write(manifests); err != nil {
-		return err
-	}
-	if err = os.Chmod(tmpFile.Name(), 0644); err != nil {
-		return err
+		return nil, nil, err
 	}
 
 	pullReader, err := cli.ImagePull(ctx, "docker.io/bitnami/kubectl:1.14", types.ImagePullOptions{})
 	if err != nil {
-		return err
+		return nil, nil, err
 	}
 	io.Copy(ioutil.Discard, pullReader)
 
@@ -111,11 +97,7 @@ func (c Cluster) apply(manifests []byte, showStdOut bool) error {
 		Env: []string{
 			"KUBECONFIG=/kubeconfig",
 		},
-		Cmd: []string{
-			"apply",
-			"-f",
-			"/manifests.yaml",
-		},
+		Cmd: cmd,
 	}
 	hostConfig := &container.HostConfig{
 		Mounts: []mount.Mount{
@@ -124,17 +106,52 @@ func (c Cluster) apply(manifests []byte, showStdOut bool) error {
 				Source: c.KubeConfigFromDockerPath,
 				Target: "/kubeconfig",
 			},
-			{
-				Type:   "bind",
-				Source: tmpFile.Name(),
-				Target: "/manifests.yaml",
-			},
 		},
 		ExtraHosts: []string{
 			"kubernetes:172.17.0.1",
 		},
 	}
 
+	return containerConfig, hostConfig, nil
+}
+
+func (c Cluster) apply(manifests []byte, showStdOut bool) error {
+	ctx := context.Background()
+
+	tmpFile, err := ioutil.TempFile(os.TempDir(), "manifests-")
+	if err != nil {
+		return err
+	}
+	defer os.Remove(tmpFile.Name())
+	if _, err = tmpFile.Write(manifests); err != nil {
+		return err
+	}
+	if err = os.Chmod(tmpFile.Name(), 0644); err != nil {
+		return err
+	}
+
+	cmd := []string{
+		"apply",
+		"-f",
+		"/manifests.yaml",
+	}
+	containerConfig, hostConfig, err := c.kubectl(ctx, cmd)
+	if err != nil {
+		return err
+	}
+
+	hostConfig.Mounts = append(hostConfig.Mounts,
+		mount.Mount{
+			Type:   "bind",
+			Source: tmpFile.Name(),
+			Target: "/manifests.yaml",
+		})
+
+	cli, err := client.NewEnvClient()
+	if err != nil {
+		return err
+	}
+
 	resp, err := cli.ContainerCreate(ctx, containerConfig, hostConfig, nil, "")
 	if err != nil {
 		return err
@@ -172,3 +189,54 @@ func (c Cluster) apply(manifests []byte, showStdOut bool) error {
 
 	return nil
 }
+
+func (c Cluster) exec(podName string, command string, args []string) (int64, []byte, []byte, error) {
+	ctx := context.Background()
+
+	cmd := []string{
+		"exec",
+		podName,
+		command,
+		"--",
+	}
+	cmd = append(cmd, args...)
+
+	containerConfig, hostConfig, err := c.kubectl(ctx, cmd)
+	if err != nil {
+		return -1, nil, nil, err
+	}
+
+	cli, err := client.NewEnvClient()
+	if err != nil {
+		return -1, nil, nil, err
+	}
+
+	resp, err := cli.ContainerCreate(ctx, containerConfig, hostConfig, nil, "")
+	if err != nil {
+		return -1, nil, nil, err
+	}
+	defer cli.ContainerRemove(ctx, resp.ID, types.ContainerRemoveOptions{})
+
+	startOptions := types.ContainerStartOptions{}
+	err = cli.ContainerStart(ctx, resp.ID, startOptions)
+	if err != nil {
+		return -1, nil, nil, err
+	}
+
+	exitCode, err := cli.ContainerWait(ctx, resp.ID)
+	if err != nil {
+		return -1, nil, nil, err
+	}
+
+	data, err := cli.ContainerLogs(ctx, resp.ID, types.ContainerLogsOptions{ShowStdout: true, ShowStderr: true})
+	if err != nil {
+		return -1, nil, nil, err
+	}
+
+	stdOut := new(bytes.Buffer)
+	stdErr := new(bytes.Buffer)
+
+	stdcopy.StdCopy(stdOut, stdErr, data)
+
+	return exitCode, stdOut.Bytes(), stdErr.Bytes(), nil
+}