Skip to content
Permalink
Browse files

Use Flyway to deploy SQL schema to non-prod (#255)

* Use Flyway to deploy SQL schema to non-prod

Added Gradle tasks to deploy and drop schema in alpha
using Flyway.

Updated ClaimsList.java so that Hibernate-generated
schema would use the right types.

Using 'varchar(255)' instead of 'text' for string columns
for now. We will need to investigate how to force Hibernate
to use the desired types in all cases.

* Use Flyway to deploy SQL schema to non-prod

Added Gradle tasks to deploy and drop schema in alpha
using Flyway.

Updated ClaimsList.java so that Hibernate-generated
schema would use the right types.

Using 'varchar(255)' instead of 'text' for string columns
for now. We will need to investigate how to force Hibernate
to use the desired types in all cases.Added Gradle tasks to deploy and drop schema in alpha
using Flyway.

Updated ClaimsList.java so that Hibernate-generated
schema would use the right types.

Using 'varchar(255)' instead of 'text' for string columns
for now. We will need to investigate how to force Hibernate
to use the desired types in all cases.

* Use Flyway to deploy SQL schema to non-prod

Added Gradle tasks to deploy and drop schema in alpha
using Flyway.

Corrected the type of ClaimsEntry's revision_id column.
It should be plain int8, not bigserial.

Make GenerateSqlSchemaCommand use a custom dialect that
converts all varchar type to 'text' and timestamp to
'timestamptz'.

* Use Flyway to deploy SQL schema to non-prod

Added Gradle tasks to deploy and drop schema in alpha
using Flyway.

Use a custome dialect in GenerateSqlSchemaCommand to
convert varchar type to 'text' and timestamp to 'timestamptz'.

Corrected ClaimsEntry's revision_id column type to int8.
This column tracks parent table's primary key and should
not be bigserial.

* Use Flyway to deploy SQL schema to non-prod

Added Gradle tasks to deploy and drop schema in alpha
using Flyway.

Use a custome dialect in GenerateSqlSchemaCommand to
convert varchar type to 'text' and timestamp to 'timestamptz'.

Corrected ClaimsEntry's revision_id column type to int8.
This column tracks parent table's primary key and should
not be bigserial.

* Use Flyway to deploy SQL schema to non-prod

Added Gradle tasks to deploy and drop schema in alpha
using Flyway.

Use a custome dialect in GenerateSqlSchemaCommand to
convert varchar type to 'text' and timestamp to 'timestamptz'.

Corrected ClaimsEntry's revision_id column type to int8.
This column tracks parent table's primary key and should
not be bigserial.
  • Loading branch information...
weiminyu committed Sep 6, 2019
1 parent ded6d38 commit 471ed7caa7d74a4e9f1cd033564e1f38666dd107
@@ -82,6 +82,8 @@ apply from: 'dependencies.gradle'

apply from: 'dependency_lic.gradle'

apply from: 'utils.gradle'

// Custom task to run checkLicense in buildSrc, which is not triggered
// by root project tasks. A shell task is used because buildSrc tasks
// cannot be referenced in the same way as tasks from a regular included
@@ -236,8 +238,9 @@ subprojects {
}
}

if (['util', 'proxy', 'core', 'prober'].contains(project.name)) return
if (['util', 'proxy', 'core', 'prober', 'db'].contains(project.name)) return

// TODO(weiminyu): investigate if the block below is still needed
ext.relativePath = "google/registry/${project.name}"

sourceSets.each {
@@ -326,20 +329,6 @@ def createGetBuildSrcDirectDepsTask(outputFileName) {
}

rootProject.ext {

// Executes an arbitrary shell command in bash and returns all output
// to stdout as a string. This method allows pipes in shell command.
execInBash = { shellCommand, bashWorkingDir ->
return new ByteArrayOutputStream().withStream { os ->
exec {
workingDir bashWorkingDir
commandLine 'bash', '-c', "${shellCommand}"
standardOutput os
}
return os
}.toString().trim()
}

invokeJavaDiffFormatScript = { action ->
def scriptDir = rootDir.path.endsWith('buildSrc')
? "${rootDir}/../java-format"
@@ -12,6 +12,9 @@
{
"moduleLicense": "Apache-2.0"
},
{
"moduleLicense": "Apache License"
},
{
"moduleLicense": "Apache License 2.0"
},
@@ -126,9 +129,18 @@
{
"moduleLicense": "Eclipse Public License v1.0"
},
{
"moduleLicense": "Eclipse Public License - v 2.0"
},
{
"moduleLicense": "https://www.eclipse.org/legal/epl-2.0/, http://www.gnu.org/copyleft/gpl.html, http://www.gnu.org/licenses/lgpl.html"
},
{
"moduleLicense": "Google App Engine Terms of Service"
},
{
"moduleLicense": "GNU General Public License Version 2"
},
{
"moduleLicense": "GNU General Public License, version 2, with the Classpath Exception"
},
@@ -144,6 +156,9 @@
{
"moduleLicense": "GNU Lesser Public License"
},
{
"moduleLicense": "GNU Lesser General Public License Version 2.1"
},
{
"moduleLicense": "GNU Library General Public License v2.1 or later"
},
@@ -79,7 +79,8 @@ def fails(self, file):
("java", "js", "soy", "sql", "py", "sh", "gradle"), {
".git", "/build/", "/generated/", "node_modules/",
"JUnitBackports.java", "registrar_bin.", "registrar_dbg.",
"google-java-format-diff.py"
"google-java-format-diff.py",
"nomulus.golden.sql"
}, REQUIRED):
"File did not include the license header.",

@@ -32,11 +32,13 @@
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.sql.Types;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.Map;
import org.hibernate.boot.MetadataSources;
import org.hibernate.boot.registry.StandardServiceRegistryBuilder;
import org.hibernate.dialect.PostgreSQL95Dialect;
import org.hibernate.tool.hbm2ddl.SchemaExport;
import org.hibernate.tool.schema.TargetType;
import org.joda.time.Period;
@@ -140,7 +142,7 @@ public void run() {
try {
// Configure Hibernate settings.
Map<String, String> settings = new HashMap<>();
settings.put("hibernate.dialect", "org.hibernate.dialect.PostgreSQL9Dialect");
settings.put("hibernate.dialect", NomulusPostgreSQLDialect.class.getName());
settings.put(
"hibernate.connection.url",
"jdbc:postgresql://" + databaseHost + ":" + databasePort + "/postgres?useSSL=false");
@@ -190,4 +192,13 @@ public void run() {
}
}
}

/** Nomulus mapping rules for column types in Postgresql. */
public static class NomulusPostgreSQLDialect extends PostgreSQL95Dialect {
public NomulusPostgreSQLDialect() {
super();
registerColumnType(Types.VARCHAR, "text");
registerColumnType(Types.TIMESTAMP, "timestamptz");
}
}
}
@@ -0,0 +1,47 @@
## Summary

This project contains Nomulus's Cloud SQL schema and schema deployment utilities.

### Schema Creation DDL

Currently we use Flywaydb for schema deployment. Versioned migration scripts
are organized in the src/main/resources/sql/flyway folder. Scripts must follow
the V{id}__{description text}.sql naming pattern (Note the double underscore).
The 'nomulus.golden.sql' file in src/main/resources/sql/schema folder is
mainly informational. It is generated by Hibernate and should not be
reformatted. We will use it in validation tests later.
### Non-production Schema Push
To manage schema in a non-production environment, use the 'flywayMigration' task.
You will need Cloud SDK and login once.
```shell
# One time login
gcloud auth login
# Deploy the current schema to alpha
gradlew :db:flywayMigrate -PdbServer=alpha
# Delete the entire schema in alpha
gradlew :db:flywayClean -PdbServer=alpha
```
The flywayMigrate task is idempotent. Repeated runs will not introduce problems.
The Flyway tasks may also be used to deploy to local instances, e.g, your own
test instance. E.g.,
```shell
# Deploy to a local instance at standard port as the super user.
gradlew :db:flywayMigrate -PdbServer=192.168.9.2 -PdbPassword=domain-registry
# Full specification of all parameters
gradlew :db:flywayMigrate -PdbServer=192.168.9.2:5432 -PdbUser=postgres \
-PdbPassword=domain-registry
```
### Production Schema Deployment
Schema deployment to production and sandbox is under development.
@@ -0,0 +1,99 @@
// Copyright 2019 The Nomulus Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

import com.google.common.collect.ImmutableList

plugins {
id "org.flywaydb.flyway" version "6.0.1"
}

ext {
def dbServerProperty = 'dbServer'
def dbNameProperty = 'dbName'

def dbServer = findProperty(dbServerProperty)
def dbName = findProperty(dbNameProperty)

getAccessInfoByHostPort = { hostAndPort ->
return [
url: "jdbc:postgresql://${hostAndPort}/${dbName}",
user: findProperty('dbUser'),
password: findProperty('dbPassword')]
}

getSocketFactoryAccessInfo = {
def cred = getCloudSqlCredential('alpha', 'superuser').split(' ')
def sqlInstance = cred[0]
return [
url: """\
jdbc:postgresql://google/${dbName}?cloudSqlInstance=
${sqlInstance}&socketFactory=
com.google.cloud.sql.postgres.SocketFactory"""
.stripIndent()
.replaceAll(System.lineSeparator(), '') ,
user: cred[1],
password: cred[2]]
}

getJdbcAccessInfo = {
switch (dbServer.toString().toLowerCase()) {
case 'alpha':
return getSocketFactoryAccessInfo()
default:
return getAccessInfoByHostPort(dbServer)
}
}

// Retrieves Cloud SQL credential for a given role. Result is in the form of
// 'instancename username password'.
//
// The env parameter may be one of the following: alpha, crash, sandbox, or
// production. The role parameter may be superuser. (More roles will be added
// later).
getCloudSqlCredential = { env, role ->
env = env == 'production' ? '' : "-${env}"
def command =
"""gsutil cp \
gs://domain-registry${env}-cloudsql-credentials/${role}.enc - | \
gcloud kms decrypt --location global --keyring nomulus \
--key sql-credentials-on-gcs-key --plaintext-file=- \
--ciphertext-file=- \
--project=domain-registry${env}-keys"""

return execInBash(command, '/tmp')
}
}

flyway {
def accessInfo = project.ext.getJdbcAccessInfo()

url = accessInfo.url
user = accessInfo.user
password = accessInfo.password
schemas = [ 'public' ]

locations = [ "classpath:sql/flyway" ]
}

dependencies {
runtimeOnly 'org.flywaydb:flyway-core:5.2.4'

runtimeOnly 'com.google.cloud.sql:postgres-socket-factory:1.0.12'
runtimeOnly 'org.postgresql:postgresql:42.2.5'
}

// Ensure that resources are rebuilt before running Flyway tasks
tasks
.findAll { task -> task.group.equals('Flyway')}
.collect { task -> task.dependsOn('buildNeeded') }
@@ -0,0 +1,6 @@
# This is a Gradle generated file for dependency locking.
# Manual edits can break the build and are not advised.
# This file is expected to be part of source control.
gradle.plugin.com.boxfuse.client:gradle-plugin-publishing:6.0.1
org.flywaydb.flyway:org.flywaydb.flyway.gradle.plugin:6.0.1
org.flywaydb:flyway-core:6.0.1
@@ -0,0 +1,31 @@
-- Copyright 2019 The Nomulus Authors. All Rights Reserved.
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.

create table "ClaimsEntry" (
revision_id int8 not null,
claim_key text not null,
domain_label text not null,
primary key (revision_id, domain_label)
);

create table "ClaimsList" (
revision_id bigserial not null,
creation_timestamp timestamptz not null,
primary key (revision_id)
);

alter table "ClaimsEntry"
add constraint FKlugn0q07ayrtar87dqi3vs3c8
foreign key (revision_id)
references "ClaimsList";
@@ -19,7 +19,7 @@ CREATE TABLE `ClaimsList` (
);

CREATE TABLE `ClaimsEntry` (
revision_id BIGSERIAL NOT NULL,
revision_id int8 NOT NULL,
claim_key TEXT NOT NULL,
domain_label TEXT NOT NULL,
PRIMARY KEY (revision_id, domain_label),

0 comments on commit 471ed7c

Please sign in to comment.
You can’t perform that action at this time.