From 6e1231233ed27df6fe70dfce813bc1877ed29fee Mon Sep 17 00:00:00 2001 From: Michael Muller Date: Tue, 10 Mar 2020 16:32:14 -0400 Subject: [PATCH] Create a nom_build wrapper script (#508) * Create a nom_build wrapper script nom_build is a wrapper around ./gradlew. It's purpose is to help us deal with properties. The main problem that it is trying to solve is that when properties are specified using -P, we don't get an error if the property we specify isn't correct. As a result, a user or a build agent can launch a build with unintended parameters. nom_build consolidates all of the properties that we define into a python script where the properties are translated to flags (actual gradlew flags are also proxied). It also generates the property file and warns the user if the current properties file is out of sync with the script and includes documentation on each of the properties. --- config/nom_build.py | 327 +++++++++++++++++++++++++++++++++++++++ config/nom_build_test.py | 109 +++++++++++++ gradle.properties | 22 ++- nom_build | 5 + 4 files changed, 450 insertions(+), 13 deletions(-) create mode 100644 config/nom_build.py create mode 100644 config/nom_build_test.py create mode 100755 nom_build diff --git a/config/nom_build.py b/config/nom_build.py new file mode 100644 index 00000000000..2ce896ddbc9 --- /dev/null +++ b/config/nom_build.py @@ -0,0 +1,327 @@ +# Copyright 2020 The Nomulus Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Script to generate dr-build and the properties file. +""" + +import argparse +import attr +import io +import os +import subprocess +import sys +from typing import List, Union + + +@attr.s(auto_attribs=True) +class Property: + name : str = '' + desc : str = '' + default : str = '' + constraints : type = str + + def validate(self, value: str): + """Verify that "value" is appropriate for the property.""" + if type is bool: + if value not in ('true', 'false'): + raise ValidationError('value of {self.name} must be "true" or ' + '"false".') + +@attr.s(auto_attribs=True) +class GradleFlag: + flags : Union[str, List[str]] + desc : str + has_arg : bool = False + + +PROPERTIES_HEADER = """\ +# This file defines properties used by the gradle build. It must be kept in +# sync with config/nom_build.py. +# +# To regenerate, run config/nom_build.py --generate-gradle-properties +# +# To view property descriptions (which are command line flags for +# nom_build), run config/nom_build.py --help. +# +# DO NOT EDIT THIS FILE BY HAND +org.gradle.jvmargs=-Xmx1024m +""" + +# Define all of our special gradle properties here. +PROPERTIES = [ + Property('mavenUrl', + 'URL to use for the main maven repository (defaults to maven ' + 'central). This can be http(s) or a "gcs" repo.'), + Property('pluginsUrl', + 'URL to use for the gradle plugins repository (defaults to maven ' + 'central, see also mavenUrl'), + Property('uploaderDestination', + 'Location to upload test reports to. Normally this should be a ' + 'GCS url (see also uploaderCredentialsFile)'), + Property('uploaderCredentialsFile', + 'json credentials file to use to upload test reports.'), + Property('uploaderMultithreadedUpload', + 'Whether to enable multithread upload.'), + Property('verboseTestOutput', + 'If true, show all test output in near-realtime.', + 'false', + bool), + Property('flowDocsFile', + 'Output filename for the flowDocsTool command.'), + Property('enableDependencyLocking', + 'Enables dependency locking.', + 'true', + bool), + Property('enableCrossReferencing', + 'generate metadata during java compile (used for kythe source ' + 'reference generation).', + 'false'), + Property('testFilter', + 'Comma separated list of test patterns, if specified run only ' + 'these.'), + Property('environment', 'GAE Environment for deployment and staging.'), + + # Cloud SQL properties + Property('dbServer', + 'A registry environment name (e.g., "alpha") or a host[:port] ' + 'string'), + Property('dbName', + 'Database name to use in connection.', + 'postgres'), + Property('dbUser', 'Database user name for use in connection'), + Property('dbPassword', 'Database password for use in connection'), + + Property('publish_repo', + 'Maven repository that hosts the Cloud SQL schema jar and the ' + 'registry server test jars. Such jars are needed for ' + 'server/schema integration tests. Please refer to integration project for more ' + 'information.'), + Property('schema_version', + 'The nomulus version tag of the schema for use in a database' + 'integration test.'), + Property('nomulus_version', + 'The version of nomulus to test against in a database ' + 'integration test.'), +] + +GRADLE_FLAGS = [ + GradleFlag(['-a', '--no-rebuild'], + 'Do not rebuild project dependencies.'), + GradleFlag(['-b', '--build-file'], 'Specify the build file.', True), + GradleFlag(['--build-cache'], + 'Enables the Gradle build cache. Gradle will try to reuse ' + 'outputs from previous builds.'), + GradleFlag(['-c', '--settings-file'], 'Specify the settings file.', True), + GradleFlag(['--configure-on-demand'], + 'Configure necessary projects only. Gradle will attempt to ' + 'reduce configuration time for large multi-project builds. ' + '[incubating]'), + GradleFlag(['--console'], + 'Specifies which type of console output to generate. Values ' + "are 'plain', 'auto' (default), 'rich' or 'verbose'.", + True), + GradleFlag(['--continue'], 'Continue task execution after a task failure.'), + GradleFlag(['-D', '--system-prop'], + 'Set system property of the JVM (e.g. -Dmyprop=myvalue).', + True), + GradleFlag(['-d', '--debug'], + 'Log in debug mode (includes normal stacktrace).'), + GradleFlag(['--daemon'], + 'Uses the Gradle Daemon to run the build. Starts the Daemon ' + 'if not running.'), + GradleFlag(['--foreground'], 'Starts the Gradle Daemon in the foreground.'), + GradleFlag(['-g', '--gradle-user-home'], + 'Specifies the gradle user home directory.', + True), + GradleFlag(['-I', '--init-script'], 'Specify an initialization script.', + True), + GradleFlag(['-i', '--info'], 'Set log level to info.'), + GradleFlag(['--include-build'], + 'Include the specified build in the composite.', + True), + GradleFlag(['-m', '--dry-run'], + 'Run the builds with all task actions disabled.'), + GradleFlag(['--max-workers'], + 'Configure the number of concurrent workers Gradle is ' + 'allowed to use.', + True), + GradleFlag(['--no-build-cache'], 'Disables the Gradle build cache.'), + GradleFlag(['--no-configure-on-demand'], + 'Disables the use of configuration on demand. [incubating]'), + GradleFlag(['--no-daemon'], + 'Do not use the Gradle daemon to run the build. Useful ' + 'occasionally if you have configured Gradle to always run ' + 'with the daemon by default.'), + GradleFlag(['--no-parallel'], + 'Disables parallel execution to build projects.'), + GradleFlag(['--no-scan'], + 'Disables the creation of a build scan. For more information ' + 'about build scans, please visit ' + 'https://gradle.com/build-scans.'), + GradleFlag(['--offline'], + 'Execute the build without accessing network resources.'), + GradleFlag(['-P', '--project-prop'], + 'Set project property for the build script (e.g. ' + '-Pmyprop=myvalue).', + True), + GradleFlag(['-p', '--project-dir'], + 'Specifies the start directory for Gradle. Defaults to ' + 'current directory.'), + GradleFlag(['--parallel'], + 'Build projects in parallel. Gradle will attempt to ' + 'determine the optimal number of executor threads to use.'), + GradleFlag(['--priority'], + 'Specifies the scheduling priority for the Gradle daemon and ' + "all processes launched by it. Values are 'normal' (default) " + "or 'low' [incubating]", + True), + GradleFlag(['--profile'], + 'Profile build execution time and generates a report in the ' + '/reports/profile directory.'), + GradleFlag(['--project-cache-dir'], + 'Specify the project-specific cache directory. Defaults to ' + '.gradle in the root project directory.', + True), + GradleFlag(['-q', '--quiet'], 'Log errors only.'), + GradleFlag(['--refresh-dependencies'], 'Refresh the state of dependencies.'), + GradleFlag(['--rerun-tasks'], 'Ignore previously cached task results.'), + GradleFlag(['-S', '--full-stacktrace'], + 'Print out the full (very verbose) stacktrace for all ' + 'exceptions.'), + GradleFlag(['-s', '--stacktrace'], + 'Print out the stacktrace for all exceptions.'), + GradleFlag(['--scan'], + 'Creates a build scan. Gradle will emit a warning if the ' + 'build scan plugin has not been applied. ' + '(https://gradle.com/build-scans)'), + GradleFlag(['--status'], + 'Shows status of running and recently stopped Gradle ' + 'Daemon(s).'), + GradleFlag(['--stop'], 'Stops the Gradle Daemon if it is running.'), + GradleFlag(['-t', '--continuous'], + 'Enables continuous build. Gradle does not exit and will ' + 're-execute tasks when task file inputs change.'), + GradleFlag(['--update-locks'], + 'Perform a partial update of the dependency lock, letting ' + 'passed in module notations change version. [incubating]'), + GradleFlag(['-v', '--version'], 'Print version info.'), + GradleFlag(['-w', '--warn'], 'Set log level to warn.'), + GradleFlag(['--warning-mode'], + 'Specifies which mode of warnings to generate. Values are ' + "'all', 'fail', 'summary'(default) or 'none'", + True), + GradleFlag(['--write-locks'], + 'Persists dependency resolution for locked configurations, ' + 'ignoring existing locking information if it exists ' + '[incubating]'), + GradleFlag(['-x', '--exclude-task'], + 'Specify a task to be excluded from execution.', + True), +] +def generate_gradle_properties() -> str: + """Returns the expected contents of gradle.properties.""" + out = io.StringIO() + out.write(PROPERTIES_HEADER) + + for prop in PROPERTIES: + out.write(f'{prop.name}={prop.default}\n') + + return out.getvalue() + + +def get_root() -> str: + """Returns the root of the nomulus build tree.""" + cur_dir = os.getcwd() + if not os.path.exists(os.path.join(cur_dir, '.git')) or \ + not os.path.exists(os.path.join(cur_dir, 'core')) or \ + not os.path.exists(os.path.join(cur_dir, 'gradle.properties')): + raise Exception('You must run this script from the root directory') + return cur_dir + + +def main(args): + parser = argparse.ArgumentParser('nom_build') + for prop in PROPERTIES: + parser.add_argument('--' + prop.name, default=prop.default, + help=prop.desc) + + # Add Gradle flags. We set 'dest' to the first flag to get a name that is + # predictable for getattr (even though it will have a leading '-' and thus + # we can't use normal python attribute syntax to get it). + for flag in GRADLE_FLAGS: + if flag.has_arg: + parser.add_argument(*flag.flags, dest=flag.flags[0], + help=flag.desc) + else: + parser.add_argument(*flag.flags, dest=flag.flags[0], + help=flag.desc, + action='store_true') + + # Add a flag to regenerate the gradle properties file. + parser.add_argument('--generate-gradle-properties', + help='Regenerate the gradle.properties file. This ' + 'file must be regenerated when changes are made to ' + 'config/nom_build.py, and should not be updated by ' + 'hand.', + action='store_true') + + # Consume the remaining non-flag arguments. + parser.add_argument('non_flag_args', nargs='*') + + # Parse command line arguments. Note that this exits the program and + # prints usage if either of the help options (-h, --help) are specified. + args = parser.parse_args(args) + + gradle_properties = generate_gradle_properties() + root = get_root() + + # If we're regenerating properties, do so and exit. + if args.generate_gradle_properties: + with open(f'{root}/gradle.properties', 'w') as dst: + dst.write(gradle_properties) + return + + # Verify that the gradle properties file is what we expect it to be. + with open(f'{root}/gradle.properties') as src: + if src.read() != gradle_properties: + print('\033[33mWARNING:\033[0m Gradle properties out of sync ' + 'with nom_build. Run with --generate-gradle-properties ' + 'to regenerate.') + + # Add properties to the gradle argument list. + gradle_command = [f'{root}/gradlew'] + for prop in PROPERTIES: + arg_val = getattr(args, prop.name) + if arg_val != prop.default: + prop.validate(arg_val) + gradle_command.extend(['-P', f'{prop.name}={arg_val}']) + + # Add Gradle flags to the gradle argument list. + for flag in GRADLE_FLAGS: + arg_val = getattr(args, flag.flags[0]) + if arg_val: + gradle_command.append(flag.flags[-1]) + if flag.has_arg: + gradle_command.append(arg_val) + + # Add the non-flag args (we exclude the first, which is the command name + # itself) and run. + gradle_command.extend(args.non_flag_args[1:]) + subprocess.call(gradle_command) + + +if __name__ == '__main__': + main(sys.argv) + diff --git a/config/nom_build_test.py b/config/nom_build_test.py new file mode 100644 index 00000000000..9745cfe7c30 --- /dev/null +++ b/config/nom_build_test.py @@ -0,0 +1,109 @@ +# Copyright 2020 The Nomulus Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import io +import os +import unittest +from unittest import mock +import nom_build +import subprocess + +FAKE_PROPERTIES = [ + nom_build.Property('foo', 'help text'), + nom_build.Property('bar', 'more text', 'true', bool), +] + +FAKE_PROP_CONTENTS = nom_build.PROPERTIES_HEADER + 'foo=\nbar=true\n' +PROPERTIES_FILENAME = '/tmp/rootdir/gradle.properties' +GRADLEW = '/tmp/rootdir/gradlew' + + +class FileFake(io.StringIO): + """File fake that writes file contents to the dictionary on close.""" + def __init__(self, contents_dict, filename): + self.dict = contents_dict + self.filename = filename + super(FileFake, self).__init__() + + def close(self): + self.dict[self.filename] = self.getvalue() + super(FileFake, self).close() + + +class MyTest(unittest.TestCase): + + def open_fake(self, filename, action='r'): + if action == 'r': + return io.StringIO(self.file_contents.get(filename, '')) + elif action == 'w': + result = self.file_contents[filename] = ( + FileFake(self.file_contents, filename)) + return result + else: + raise Exception(f'Unexpected action {action}') + + def print_fake(self, data): + self.printed.append(data) + + def setUp(self): + self.addCleanup(mock.patch.stopall) + self.exists_mock = mock.patch.object(os.path, 'exists').start() + self.getcwd_mock = mock.patch.object(os, 'getcwd').start() + self.getcwd_mock.return_value = '/tmp/rootdir' + self.open_mock = ( + mock.patch.object(nom_build, 'open', self.open_fake).start()) + self.print_mock = ( + mock.patch.object(nom_build, 'print', self.print_fake).start()) + + self.call_mock = mock.patch.object(subprocess, 'call').start() + + self.file_contents = { + # Prefil with the actual file contents. + PROPERTIES_FILENAME: nom_build.generate_gradle_properties() + } + self.printed = [] + + @mock.patch.object(nom_build, 'PROPERTIES', FAKE_PROPERTIES) + def test_property_generation(self): + self.assertEqual(nom_build.generate_gradle_properties(), + FAKE_PROP_CONTENTS) + + @mock.patch.object(nom_build, 'PROPERTIES', FAKE_PROPERTIES) + def test_property_file_write(self): + nom_build.main(['nom_build', '--generate-gradle-properties']) + self.assertEqual(self.file_contents[PROPERTIES_FILENAME], + FAKE_PROP_CONTENTS) + + def test_property_file_incorrect(self): + self.file_contents[PROPERTIES_FILENAME] = 'bad contents' + nom_build.main(['nom_build']) + self.assertIn('', self.printed[0]) + + def test_no_args(self): + nom_build.main(['nom_build']) + self.assertEqual(self.printed, []) + self.call_mock.assert_called_with([GRADLEW]) + + def test_property_calls(self): + nom_build.main(['nom_build', '--testFilter=foo']) + self.call_mock.assert_called_with([GRADLEW, '-P', 'testFilter=foo']) + + def test_gradle_flags(self): + nom_build.main(['nom_build', '-d', '-b', 'foo']) + self.call_mock.assert_called_with([GRADLEW, '--build-file', 'foo', + '--debug']) + +unittest.main() + + diff --git a/gradle.properties b/gradle.properties index c5833626d55..6400bcb3179 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,12 @@ +# This file defines properties used by the gradle build. It must be kept in +# sync with config/nom_build.py. +# +# To regenerate, run config/nom_build.py --generate-gradle-properties +# +# To view property descriptions (which are command line flags for +# nom_build), run config/nom_build.py --help. +# +# DO NOT EDIT THIS FILE BY HAND org.gradle.jvmargs=-Xmx1024m mavenUrl= pluginsUrl= @@ -8,25 +17,12 @@ verboseTestOutput=false flowDocsFile= enableDependencyLocking=true enableCrossReferencing=false - -# Comma separated list of test patterns, if specified run only these. testFilter= - -# GAE Environment for deployment and staging. environment= - -# Cloud SQL properties - -# A registry environment name (e.g., 'alpha') or a host[:port] string dbServer= dbName=postgres dbUser= dbPassword= - -# Maven repository that hosts the Cloud SQL schema jar and the registry -# server test jars. Such jars are needed for server/schema integration tests. -# Please refer to integration project -# for more information. publish_repo= schema_version= nomulus_version= diff --git a/nom_build b/nom_build new file mode 100755 index 00000000000..ae69b4e28ba --- /dev/null +++ b/nom_build @@ -0,0 +1,5 @@ +#!/bin/sh +# Wrapper for nom_build.py. +cd $(dirname $0) +python3 ./config/nom_build.py "$@" +exit $?