Skip to content
Browse files

Run cross-release SQL integration tests (#403)

* Run cross-release SQL integration tests

Run SQL integration tests across arbitrary schema and server

Refer to integration/ in this change for more information.

TESTED=Cloud build changes tested with cloud-build-local
       Used the published jars to test sqlIntegration task locally.
  • Loading branch information
weiminyu committed Dec 12, 2019
1 parent 9853f23 commit f48e3933f5804a675cb77d4f1b406d1ec4ae5749
@@ -221,9 +221,18 @@ subprojects {

afterEvaluate {
if (rootProject.enableDependencyLocking.toBoolean()) {
// Lock application dependencies except for the gradle-license-report
// plugin. See dependency_lic.gradle for the reason why.
if (rootProject.enableDependencyLocking.toBoolean()
&& != 'integration') {
// The ':integration' project runs server/schema integration tests using
// dynamically specified jars with no transitive dependency. Therefore
// dependency-locking does not make sense. Furthermore, during
// evaluation it resolves the 'testRuntime' configuration, making it
// immutable. Locking activation would trigger an invalid operation
// exception.
// For all other projects, due to problem with the gradle-license-report
// plugin, the dependencyLicenseReport configuration must opt out of
// dependency-locking. See dependency_lic.gradle for the reason why.
// To selectively activate dependency locking without hardcoding them
// in the 'configurations' block, the following code must run after
@@ -16,6 +16,7 @@ import java.lang.reflect.Constructor

plugins {
id 'java-library'
id 'maven-publish'

// Path to code generated by ad hoc tasks in this project. A separate path is
@@ -112,6 +113,11 @@ configurations {

// Published jars that are used for server/schema compatibility tests.
// See <a href="../integration/">the integration project</a>
// for details.

// Exclude non-canonical servlet-api jars. Our AppEngine deployment uses
// javax.servlet:servlet-api:2.5
// For reasons we do not understand, marking the following dependencies as
@@ -814,6 +820,66 @@ test {
createUberJar('nomulus', 'nomulus', '')
project.nomulus.dependsOn project(':third_party').jar

// A jar with classes and resources from main sourceSet, excluding internal
// data. See comments on configurations.nomulus_test above for details.
task nomulusFossJar (type: Jar) {
archiveBaseName = 'nomulus'
archiveClassifier = 'public'
from (project.sourceSets.main.output) {
exclude 'google/registry/config/files/**'
from (project.sourceSets.main.output) {
include 'google/registry/config/files/default-config.yaml'
include 'google/registry/config/files/nomulus-config-unittest.yaml'

// An UberJar of registry test classes, resources and all dependencies.
// See comments on configurations.nomulus_test above for details.
// TODO(weiminyu): extract shareable code with root.ext.createUberJar
task testUberJar (
type: com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar) {
archiveBaseName = 'nomulus-tests'
archiveClassifier = 'alldeps'
zip64 = true
archiveVersion = ''
configurations = [project.configurations.testRuntimeClasspath]
from project.sourceSets.test.output
// Excludes signature files that accompany some dependency jars, like
// bonuncycastle. If they are present, only classes from those signed jars are
// made available to the class loader.
// see
exclude "META-INF/*.SF", "META-INF/*.DSA", "META-INF/*.RSA"
// Exclude SQL schema, which is a test dependency.
exclude 'sql/flyway/**'
// ShadowJar includes java source files when used on sourceSets.test.output.
// They are not needed.
exclude '**/*.java'

artifacts {
nomulus_test nomulusFossJar
nomulus_test testUberJar

publishing {
repositories {
maven {
url project.publish_repo
publications {
nomulusTestsPublication(MavenPublication) {
groupId 'google.registry'
artifactId 'nomulus_test'
version project.nomulus_version
artifact nomulusFossJar
artifact testUberJar

task buildToolImage(dependsOn: nomulus, type: Exec) {
commandLine 'docker', 'build', '-t', 'nomulus-tool', '.'
@@ -85,6 +85,7 @@
private static final String POSTGRES_DB_NAME = "postgres";

// Name of the optional property that specifies the root path of the golden schema.
// TODO(weiminyu): revert this. The :integration project offers a better solution.
static final String GOLDEN_SCHEMA_RESOURCE_ROOT_PROP = "sql_schema_resource_root";

@@ -106,17 +106,18 @@ task compileApiJar(type: Jar) {

configurations {

artifacts {
archives schemaJar
compileApi compileApiJar
schema schemaJar

publishing {
repositories {
maven {
url project.schema_publish_repo
url project.publish_repo
publications {
@@ -22,7 +22,10 @@ dbName=postgres

# Maven repository of the Cloud SQL schema jar, which contains the
# SQL DDL scripts.
# Maven repository that hosts the Cloud SQL schema jar and the registry
# server test jars. Such jars are needed for server/schema integration tests.
# Please refer to <a href="./integration/">integration project</a>
# for more information.
@@ -0,0 +1,120 @@
## Summary

This project runs cross-version server/schema integration tests with arbitrary
version pairs. It may be used by presubmit tests and continuous-integration
tests, or as a gating test during release and/or deployment.

## Maven Dependencies

This release process is expected to publish the following Maven dependencies to
a well-known repository:

* google.registry:schema, which contains the schema DDL scripts. This is done
by the ':db:publish' task.
* google.registry:nomulus_test, which contains the nomulus classes and
dependencies needed for the integration tests. This is done by the
':core:publish' task.

After each deployment in sandbox or production, the deployment process is
expected to save the version tag of the binary or schema along with the
environment. These tags will be made available to test runners.

## Usage

The ':integration:sqlIntegrationTest' task is the test runner. It uses the
following properties:

* nomulus_version: a Registry server release tag, or 'local' if the code in
the local Git tree should be used.
* schema_version: a schema release tag, or 'local' if the code in the local
Git tree should be used.
* publish_repo: the Maven repository where release jars may be found. This is
required if neither of the above is 'local'.

Given a program 'fetch_version_tag' that retrieves the currently deployed
version tag of SQL schema or server binary in a particular environment (which as
mentioned earlier are saved by the deployment process), the following code
snippet checks if the current PR or local clone has schema changes, and if yes,
tests the production server's version with the new schema.

current_prod_schema=$(fetch_version_tag schema production)
current_prod_server=$(fetch_version_tag server production)
schema_changes=$(git diff ${current_prod_schema} --name-only \
./db/src/main/resources/sql/flyway/ | wc -l)
[[ schema_changes -gt 0 ]] && ./gradlew :integration:sqlIntegrationTest \
-Ppublish_repo=${REPO} -Pschema_version=local \

## Implementation Notes

### Run Tests from Jar

Gradle test runner does not look for runnable tests in jars. We must extract
tests to a directory. For now, only the SqlIntegrationTestSuite.class needs to
be extracted. Gradle has no trouble finding its member classes.

### Hibernate Behavior

If all :core classes (main and test) and dependencies are assembled in a single
jar, Hibernate behaves strangely: every time an EntityManagerFactory is created,
regardless of what Entity classes are specified, Hibernate would scan the entire
jar for all Entity classes and complain about duplicate mapping (due to the
TestEntity classes declared by tests).

We worked around this problem by creating two jars from :core:

* The nomulus-public.jar: contains the classes and resources in the main
sourceSet (and excludes internal files under the config package).
* The nomulus-tests-alldeps.jar: contains the test classes as well as all

## Alternatives Tried

### Use Git Branches

One alternative is to rely on Git branches to set up the classes. For example,
the shell snippet shown earlier can be implemented as:

current_prod_schema=$(fetch_version_tag schema production)
current_prod_server=$(fetch_version_tag server production)
schema_changes=$(git diff ${current_prod_schema} --name-only \
./db/src/main/resources/sql/flyway/ | wc -l)
if [[ schema_changes -gt 0 ]]; then
current_branch=$(git rev-parse --abbrev-ref HEAD)
schema_folder=$(mktemp -d)
./gradlew :db:schemaJar && cp ./db/build/libs/schema.jar ${schema_folder}
git checkout ${current_prod_server}
./gradlew sqlIntegrationTest \
git checkout ${current_branch}

The drawbacks of this approach include:

* Switching branches back and forth is error-prone and risky, especially when
we run this as a gating test during release.
* Switching branches makes implicit assumptions on how the test platform would
check out the repository (e.g., whether we may be on a headless branch when
we switch).
* The generated jar is not saved, making it harder to troubleshoot.
* To use this locally during development, the Git tree must not have
uncommitted changes.

### Smaller Jars

Another alternative follows the same idea as our current approach. However,
instead of including dependencies in a fat jar, it simply records their versions
in a file. At testing time these dependencies will be imported into the gradle
project file with forced resolution (e.g., testRuntime ('junit:junit:4.12)'
{forced = true} ). This way the published jars will be smaller.

This approach conflicts with our current dependency-locking processing. Due to
issues with the license-check plugin, dependency-locking is activated after all
projects are evaluated. This approach will resolve some configurations in :core
(and make them immutable) during evaluation, causing the lock-activation (which
counts as a mutation) call to fail.
@@ -0,0 +1,99 @@
// Copyright 2019 The Nomulus Authors. All Rights Reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

// This source-less project is used to run cross-release server/SQL integration
// tests. See the file in this folder for more information.

import static
import static

if (schema_version == '' || nomulus_version == '') {

def USE_LOCAL = 'local'

if (schema_version != USE_LOCAL || nomulus_version != USE_LOCAL) {
'The publish_repo is required when remote jars are needed.')

repositories {
maven {
url project.publish_repo

def testUberJarName = ''

dependencies {
gradleLint.ignore('unused-dependency') {
if (schema_version == USE_LOCAL) {
testRuntime project(path: ':db', configuration: 'schema')
} else {
testRuntime "google.registry:schema:${schema_version}"
if (nomulus_version == USE_LOCAL) {
testRuntime project(path: ':core', configuration: 'nomulus_test')
testUberJarName = 'nomulus-tests-alldeps.jar'
} else {
testRuntime "google.registry:nomulus_test:${nomulus_version}:public"
testRuntime "google.registry:nomulus_test:${nomulus_version}:alldeps"
testUberJarName = "nomulus_test-${nomulus_version}-alldeps.jar"

configurations.testRuntime.transitive = false

def unpackedTestDir = "${projectDir}/build/unpackedTests/${nomulus_version}"

// Extracts SqlIntegrationTestSuite.class to a temp folder. Gradle's test
// runner only looks for runnable tests on a regular file system. However,
// it can load member classes of test suites from jars.
task extractSqlIntegrationTestSuite (type: Copy) {
doFirst {
outputs.dir unpackedTestDir
from zipTree(
.filter { == testUberJarName}
.singleFile).matching {
include 'google/registry/**/SqlIntegrationTestSuite.class'
into unpackedTestDir
includeEmptyDirs = false

// TODO(weiminyu): inherit from FilteringTest (defined in :core).
task sqlIntegrationTest(type: Test) {
testClassesDirs = files(unpackedTestDir)
classpath = configurations.testRuntime
include 'google/registry/schema/integration/SqlIntegrationTestSuite.*'

dependsOn extractSqlIntegrationTestSuite

finalizedBy tasks.create('removeUnpackedTests') {
doLast {
delete file(unpackedTestDir)

// Disable incremental build/test since Gradle cannot detect changes
// in dependencies on its own. Will not fix since this test is typically
// run once (in presubmit or ci tests).
outputs.upToDateWhen { false }

0 comments on commit f48e393

Please sign in to comment.
You can’t perform that action at this time.