From 333e976bd062361f65fdea60b0c8d4f320ed62f2 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Tue, 11 Jul 2017 12:51:25 -0400 Subject: [PATCH 01/12] Bumping to 2.9.0 --- gradle.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle.properties b/gradle.properties index 744ab35e3..081850c58 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ group=com.marklogic -version=2.8.0 -mlAppDeployerDependency=com.marklogic:ml-app-deployer:2.8.0 +version=2.9.0 +mlAppDeployerDependency=com.marklogic:ml-app-deployer:2.9.0 mlcpUtilDependency=com.marklogic:mlcp-util:0.3.0 From 172be9467bd67420ad87ad8725320e03d5081a93 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Tue, 11 Jul 2017 12:52:50 -0400 Subject: [PATCH 02/12] #210 New task for copying Roxy source files --- build.gradle | 5 ++- .../marklogic/gradle/MarkLogicPlugin.groovy | 6 +++ .../gradle/task/roxy/RoxyCopyFilesTask.groovy | 41 +++++++++++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 src/main/groovy/com/marklogic/gradle/task/roxy/RoxyCopyFilesTask.groovy diff --git a/build.gradle b/build.gradle index bb5ff872e..7ea7766db 100644 --- a/build.gradle +++ b/build.gradle @@ -17,6 +17,8 @@ plugins { id "eclipse" id "idea" id "com.jfrog.bintray" version "1.6" + id "com.github.jk1.dependency-license-report" version "0.3.11" + id "com.gradle.plugin-publish" version "0.9.7" } apply plugin: "com.gradle.plugin-publish" @@ -37,6 +39,7 @@ dependencies { compile localGroovy() compile mlAppDeployerDependency compile mlcpUtilDependency + compile group: 'commons-io', name: 'commons-io', version: '2.5' } task sourcesJar(type: Jar, dependsOn: classes) { @@ -94,4 +97,4 @@ pluginBundle { mavenCoordinates { version = "2.8.0" } -} +} \ No newline at end of file diff --git a/src/main/groovy/com/marklogic/gradle/MarkLogicPlugin.groovy b/src/main/groovy/com/marklogic/gradle/MarkLogicPlugin.groovy index d4c509c28..6c7c9e556 100644 --- a/src/main/groovy/com/marklogic/gradle/MarkLogicPlugin.groovy +++ b/src/main/groovy/com/marklogic/gradle/MarkLogicPlugin.groovy @@ -33,6 +33,8 @@ import com.marklogic.gradle.task.groups.SetTraceEventsTask import com.marklogic.gradle.task.mimetypes.DeployMimetypesTask import com.marklogic.gradle.task.qconsole.ExportWorkspacesTask import com.marklogic.gradle.task.qconsole.ImportWorkspacesTask +import com.marklogic.gradle.task.roxy.RoxyCopyFilesTask +import com.marklogic.gradle.task.roxy.RoxyCopyPropertiesTask import com.marklogic.gradle.task.scaffold.GenerateScaffoldTask import com.marklogic.gradle.task.schemas.LoadSchemasTask import com.marklogic.gradle.task.security.* @@ -207,6 +209,10 @@ class MarkLogicPlugin implements Plugin { String shellGroup = "ml-gradle Shell" project.task("mlShell", type: ShellTask, group: shellGroup, description: "Run groovysh with MarkLogic-specific support built in") + String roxyGroup = "ml-gradle Roxy"; + project.task("mlRoxyCopyProperties", type: RoxyCopyPropertiesTask, group: roxyGroup, description: "Copy Roxy properties to gradle.properties file") + project.task("mlRoxyCopyFiles", type: RoxyCopyFilesTask, group: roxyGroup, description: "Copy roxy files") + logger.info("Finished initializing ml-gradle\n") } diff --git a/src/main/groovy/com/marklogic/gradle/task/roxy/RoxyCopyFilesTask.groovy b/src/main/groovy/com/marklogic/gradle/task/roxy/RoxyCopyFilesTask.groovy new file mode 100644 index 000000000..6b609141a --- /dev/null +++ b/src/main/groovy/com/marklogic/gradle/task/roxy/RoxyCopyFilesTask.groovy @@ -0,0 +1,41 @@ +package com.marklogic.gradle.task.roxy + +import com.marklogic.gradle.task.MarkLogicTask +import org.apache.commons.io.FileUtils +import org.gradle.api.tasks.TaskAction + +class RoxyCopyFilesTask extends MarkLogicTask { + + def roxyFolderMapping = [ + "src" : "/root", + "rest-api/config" : "/options", + "rest-api/ext" : "/services", + "rest-api/transforms" : "/transforms" + ] + + @TaskAction + void copyRoxyFiles() { + if (getRoxyHome()) { + def baseDir = getAppConfig().getModulePaths().get(0) + roxyFolderMapping.each { k, v -> + def source = getRoxyHome() + "/" + k + println "Source folder '" + source + "' ... " + def sourceFolder = new File(source) + if (sourceFolder.exists() && sourceFolder.isDirectory()) { + def targetDir = baseDir + v + println "Creating folder '" + targetDir + "' ... " + def targetFolder = new File(targetDir) + FileUtils.forceMkdir(targetFolder) + println "Copying contents of '" + source + "' to '" + targetDir + "' ... " + FileUtils.copyDirectory(sourceFolder, targetFolder) + } + } + } else { + println "mlRoxyHome parameter is not provided. Please run using -PmlRoxyHome=/your/roxy/project/home" + } + } + + String getRoxyHome(){ + project.hasProperty("mlRoxyHome") ? project.property("mlRoxyHome") : "" + } +} From c6aa778d9a84e36ea4f60a5e36a37c60ec0c0407 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Tue, 11 Jul 2017 12:56:57 -0400 Subject: [PATCH 03/12] Merging in RoxyCopyPropertiesTask from dev --- .../gradle/task/MarkLogicTask.groovy | 12 +++-- .../task/roxy/RoxyCopyPropertiesTask.groovy | 52 +++++++++++++++++++ 2 files changed, 60 insertions(+), 4 deletions(-) create mode 100644 src/main/groovy/com/marklogic/gradle/task/roxy/RoxyCopyPropertiesTask.groovy diff --git a/src/main/groovy/com/marklogic/gradle/task/MarkLogicTask.groovy b/src/main/groovy/com/marklogic/gradle/task/MarkLogicTask.groovy index d6d843246..cd8dccb6b 100644 --- a/src/main/groovy/com/marklogic/gradle/task/MarkLogicTask.groovy +++ b/src/main/groovy/com/marklogic/gradle/task/MarkLogicTask.groovy @@ -44,10 +44,14 @@ class MarkLogicTask extends DefaultTask { project.hasProperty("mlAdminPassword") ? project.property("mlAdminPassword") : project.property("mlPassword") } + String getRoxyHome(){ + project.hasProperty("mlRoxyHome") ? project.property("mlRoxyHome") : null + } + DatabaseClient newClient() { getAppConfig().newDatabaseClient() } - + void deployWithCommandListProperty(String propertyName) { deployWithCommands(getProject().property(propertyName)) } @@ -57,7 +61,7 @@ class MarkLogicTask extends DefaultTask { deployer.setCommands(commands) deployer.deploy(getAppConfig()) } - + void undeployWithCommandListProperty(String propertyName) { undeployWithCommands(getProject().property(propertyName)) } @@ -67,12 +71,12 @@ class MarkLogicTask extends DefaultTask { deployer.setCommands(commands) deployer.undeploy(getAppConfig()) } - + void invokeDeployerCommandWithClassName(String className) { SimpleAppDeployer d = (SimpleAppDeployer)getAppDeployer() new SimpleAppDeployer(getManageClient(), getAdminManager(), d.getCommand(className)).deploy(getAppConfig()) } - + void undeployWithCommandWithClassName(String className) { SimpleAppDeployer d = (SimpleAppDeployer)getAppDeployer() new SimpleAppDeployer(getManageClient(), getAdminManager(), d.getCommand(className)).undeploy(getAppConfig()) diff --git a/src/main/groovy/com/marklogic/gradle/task/roxy/RoxyCopyPropertiesTask.groovy b/src/main/groovy/com/marklogic/gradle/task/roxy/RoxyCopyPropertiesTask.groovy new file mode 100644 index 000000000..152512e6f --- /dev/null +++ b/src/main/groovy/com/marklogic/gradle/task/roxy/RoxyCopyPropertiesTask.groovy @@ -0,0 +1,52 @@ +package com.marklogic.gradle.task.roxy + +import com.marklogic.gradle.task.MarkLogicTask +import org.gradle.api.tasks.TaskAction + +class RoxyCopyPropertiesTask extends MarkLogicTask { + + def roxyPropertyFiles = ["default.properties", "build.properties"] + def roxyGradleMapping = ["app-name" : "mlAppName", "app-port" : "mlRestPort", + "user" : "mlRestAdminUsername", "password" : "mlAdminPassword", + "content-forests-per-host" : "mlContentForestsPerHost", "group" : "mlGroupName", + "forest-data-dir": "envForestDataDirectory"] + + @TaskAction + void copyProperties() { + if (null != getRoxyHome()) { + Map roxyProperties = new LinkedHashMap() + roxyPropertyFiles.each { propertyFile -> + new File(getRoxyHome() + "/deploy/", propertyFile).eachLine { line -> + if (!line.startsWith("#")) { + def keyValue = line.split("=") + if(keyValue.length == 2) + roxyProperties.put(keyValue[0], keyValue[1]) + } + } + } + writeFile("gradle.properties", constructText(roxyProperties)) + }else{ + println "mlRoxyHome parameter is not provided. Please run using -P mlRoxyHome=/your/project/home"; + } + } + + String constructText(Map roxyProperties){ + def properties = new StringBuilder() + roxyGradleMapping.each { entry -> + def val = roxyProperties.get(entry.key) + if(null != val) + properties.append(entry.value).append("=").append(val).append("\n") + } + return properties + } + + void writeFile(String filename, String text) { + File file = new File(filename); + if (file.exists()) { + new File("backup-" + filename).write(file.text) + } + println "Writing: " + filename + file.write(text) + } + +} From b89c9c7dbf07d8a14c43944318b045d970068c03 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Sun, 16 Jul 2017 11:10:57 -0400 Subject: [PATCH 04/12] #205 New task for migrating Roxy build steps #205 Renaming Roxy tasks to use Migrate instead of Copy Mapping a few more Roxy properties --- .gitignore | 71 +++---- examples/roxy-project/.gitignore | 8 + examples/roxy-project/build.gradle | 24 +-- examples/roxy-project/deploy/app_specific.rb | 85 ++++++++ .../marklogic/gradle/MarkLogicPlugin.groovy | 16 +- .../gradle/task/MarkLogicTask.groovy | 6 +- .../gradle/task/ServerEvalTask.groovy | 28 +++ .../task/roxy/RoxyCopyPropertiesTask.groovy | 52 ----- .../roxy/RoxyMigrateBuildStepsTask.groovy | 183 ++++++++++++++++++ ...ask.groovy => RoxyMigrateFilesTask.groovy} | 21 +- .../roxy/RoxyMigratePropertiesTask.groovy | 93 +++++++++ .../gradle/task/roxy/RoxyTask.groovy | 14 ++ 12 files changed, 481 insertions(+), 120 deletions(-) create mode 100644 examples/roxy-project/.gitignore create mode 100755 examples/roxy-project/deploy/app_specific.rb create mode 100644 src/main/groovy/com/marklogic/gradle/task/ServerEvalTask.groovy delete mode 100644 src/main/groovy/com/marklogic/gradle/task/roxy/RoxyCopyPropertiesTask.groovy create mode 100644 src/main/groovy/com/marklogic/gradle/task/roxy/RoxyMigrateBuildStepsTask.groovy rename src/main/groovy/com/marklogic/gradle/task/roxy/{RoxyCopyFilesTask.groovy => RoxyMigrateFilesTask.groovy} (54%) create mode 100644 src/main/groovy/com/marklogic/gradle/task/roxy/RoxyMigratePropertiesTask.groovy create mode 100644 src/main/groovy/com/marklogic/gradle/task/roxy/RoxyTask.groovy diff --git a/.gitignore b/.gitignore index 13b4a4d4b..f1283d08d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,35 +1,36 @@ -/bin -/.classpath -/build -/.gradle -/.nb-gradle -/.settings -/.project -ml-gradle-all/build/ -ml-gradle-all/.gradle/ -ml-gradle-all/.classpath -ml-gradle-all/.project -ml-gradle-all/.settings/ -sample-project/rob/ -sample-project/data/export/ -disconnected-project/build/ -disconnected-project/.gradle/ -sample-project/local.gradle -examples/disconnected-project/build/ -examples/disconnected-project/.gradle/ -examples/flexrep-project/.gradle/ -examples/sample-project/local.gradle -examples/failover-project/.gradle/ -examples/mlcp-project/.gradle/ -examples/mlcp-project/build/ -examples/mlcp-project/data/export/ -examples/shell-project/.gradle/ -examples/shell-project/build/ -examples/shell-project/.gradle -examples/shell-project/.settings -examples/shell-project/.project -examples/shell-project/.classpath -examples/minimal-project/.gradle -examples/ignore-resources-project/.gradle/ -.idea -*.iml +/bin +/.classpath +/build +/.gradle +/.nb-gradle +/.settings +/.project +ml-gradle-all/build/ +ml-gradle-all/.gradle/ +ml-gradle-all/.classpath +ml-gradle-all/.project +ml-gradle-all/.settings/ +sample-project/rob/ +sample-project/data/export/ +disconnected-project/build/ +disconnected-project/.gradle/ +sample-project/local.gradle +examples/disconnected-project/build/ +examples/disconnected-project/.gradle/ +examples/flexrep-project/.gradle/ +examples/sample-project/local.gradle +examples/failover-project/.gradle/ +examples/mlcp-project/.gradle/ +examples/mlcp-project/build/ +examples/mlcp-project/data/export/ +examples/shell-project/.gradle/ +examples/shell-project/build/ +examples/shell-project/.gradle +examples/shell-project/.settings +examples/shell-project/.project +examples/shell-project/.classpath +examples/minimal-project/.gradle +examples/ignore-resources-project/.gradle/ +.idea +*.iml +out diff --git a/examples/roxy-project/.gitignore b/examples/roxy-project/.gitignore new file mode 100644 index 000000000..699bbc7ff --- /dev/null +++ b/examples/roxy-project/.gitignore @@ -0,0 +1,8 @@ +/bin +.classpath +.project +/build +.gradle +.settings +src +backup-* diff --git a/examples/roxy-project/build.gradle b/examples/roxy-project/build.gradle index 97c75ae1b..e8a65d648 100644 --- a/examples/roxy-project/build.gradle +++ b/examples/roxy-project/build.gradle @@ -1,14 +1,16 @@ -plugins { - id "com.marklogic.ml-gradle" version "2.7.1" +buildscript { + repositories { + mavenLocal() + jcenter() + } + dependencies { + classpath "com.marklogic:ml-gradle:2.9.0" + } } -/** - * ml-gradle defaults to a modules path of src/main/ml-modules. Roxy applications almost always store - * asset modules under ./src, so we need to modify the module paths. Note that since Roxy stores REST API - * extensions under ./ext, "gradle mlWatch" will not load these. - */ -ext { - mlAppConfig { - modulePaths = ["src"] - } +apply plugin: "com.marklogic.ml-gradle" + +repositories { + mavenLocal() + jcenter() } diff --git a/examples/roxy-project/deploy/app_specific.rb b/examples/roxy-project/deploy/app_specific.rb new file mode 100755 index 000000000..773812697 --- /dev/null +++ b/examples/roxy-project/deploy/app_specific.rb @@ -0,0 +1,85 @@ +# +# Put your custom functions in this class in order to keep the files under lib untainted +# +# This class has access to all of the private variables in deploy/lib/server_config.rb +# +# any public method you create here can be called from the command line. See +# the examples below for more information. +# +class ServerConfig + + def hello_world() + @logger.info "trying to run a custom query" + r = execute_query %Q{ + xquery version "1.0-ml"; + ("one", "two", "three") ! element span { . } + }, + { + :app_name => "#{@properties['ml.app-name']}" + } + JSON.parse(r.body).each do |item| + output = item['result'] + @logger.info " " + output + end + end + + def get_document_count() + r = execute_query %Q{ + xdmp:estimate(fn:doc()) + } + @logger.info(r.body) + end + + def delete_view() + r = execute_query %Q{ + xquery version "1.0-ml"; + + import module namespace view = "http://marklogic.com/xdmp/view" + at "/MarkLogic/views.xqy"; + + try { + view:remove( + "main", + "Compliance" + ) + } catch ($e) { () } + (: Deletes a view, of the 'main' schema that contains columns, with a scope on the element, 'html'. :) + }, + { :db_name => @properties["ml.content-db"] } + end + + def create_view() + r = execute_query %Q{ + xquery version "1.0-ml"; + + import module namespace view = "http://marklogic.com/xdmp/view" + at "/MarkLogic/views.xqy"; + + try { + view:schema-create( + "main", + () + ) + } catch ($e) {()}, + view:create( + "main", + "Compliance", + view:element-view-scope(fn:QName("http://www.w3.org/1999/xhtml","html")), + ( view:column("uri", cts:uri-reference()), + view:column("entityName", cts:path-reference('/xhtml:html/xhtml:head/xhtml:meta[@name eq "entityName"]/@content',("collation=http://marklogic.com/collation/"))), + view:column("entityStreetAddress", cts:path-reference('/xhtml:html/xhtml:head/xhtml:meta[@name eq "entityStreetAddress"]/@content',("collation=http://marklogic.com/collation/", ("nullable")))), + view:column("entityCityAddress", cts:path-reference('/xhtml:html/xhtml:head/xhtml:meta[@name eq "entityCityAddress"]/@content',("collation=http://marklogic.com/collation/", ("nullable")))), + view:column("entityCountryAddress", cts:path-reference('/xhtml:html/xhtml:head/xhtml:meta[@name eq "entityCountryAddress"]/@content',("collation=http://marklogic.com/collation//S2", ("nullable")))), + view:column("foreignEntityStatus", cts:path-reference('/xhtml:html/xhtml:head/xhtml:meta[@name eq "foreignEntityStatus"]/@content',("collation=http://marklogic.com/collation/", ("nullable")))), + view:column("intermediaryEntityStatus", cts:path-reference('/xhtml:html/xhtml:head/xhtml:meta[@name eq "intermediaryEntityStatus"]/@content',("collation=http://marklogic.com/collation/codepoint", ("nullable")))), + view:column("EIN", cts:path-reference('/xhtml:html/xhtml:head/xhtml:meta[@name eq "EIN"]/@content',("collation=http://marklogic.com/collation/", ("nullable")))), + view:column("docType", cts:path-reference('/xhtml:html/xhtml:head/xhtml:meta[@name eq "docType"]/@content',("collation=http://marklogic.com/collation//S1", ("nullable")))) + ), + () + ) + + (: Creates a view, of the 'main' schema that contains columns, with a scope on the element, 'html'. :) + }, + { :db_name => @properties["ml.content-db"] } + end +end diff --git a/src/main/groovy/com/marklogic/gradle/MarkLogicPlugin.groovy b/src/main/groovy/com/marklogic/gradle/MarkLogicPlugin.groovy index 6c7c9e556..838bfdd63 100644 --- a/src/main/groovy/com/marklogic/gradle/MarkLogicPlugin.groovy +++ b/src/main/groovy/com/marklogic/gradle/MarkLogicPlugin.groovy @@ -33,8 +33,9 @@ import com.marklogic.gradle.task.groups.SetTraceEventsTask import com.marklogic.gradle.task.mimetypes.DeployMimetypesTask import com.marklogic.gradle.task.qconsole.ExportWorkspacesTask import com.marklogic.gradle.task.qconsole.ImportWorkspacesTask -import com.marklogic.gradle.task.roxy.RoxyCopyFilesTask -import com.marklogic.gradle.task.roxy.RoxyCopyPropertiesTask +import com.marklogic.gradle.task.roxy.RoxyMigrateBuildStepsTask +import com.marklogic.gradle.task.roxy.RoxyMigrateFilesTask +import com.marklogic.gradle.task.roxy.RoxyMigratePropertiesTask import com.marklogic.gradle.task.scaffold.GenerateScaffoldTask import com.marklogic.gradle.task.schemas.LoadSchemasTask import com.marklogic.gradle.task.security.* @@ -210,9 +211,14 @@ class MarkLogicPlugin implements Plugin { project.task("mlShell", type: ShellTask, group: shellGroup, description: "Run groovysh with MarkLogic-specific support built in") String roxyGroup = "ml-gradle Roxy"; - project.task("mlRoxyCopyProperties", type: RoxyCopyPropertiesTask, group: roxyGroup, description: "Copy Roxy properties to gradle.properties file") - project.task("mlRoxyCopyFiles", type: RoxyCopyFilesTask, group: roxyGroup, description: "Copy roxy files") - + project.task("mlRoxyMigrateBuildSteps", type: RoxyMigrateBuildStepsTask, group: roxyGroup, description: "Migrate build steps from deploy/app_specific.rb into custom Gradle tasks. " + + "Use -ProxyProjectPath to define the location of your Roxy project, and -PappSpecificPath to define a path other than deploy/app_specific.rb") + project.task("mlRoxyMigrateFiles", type: RoxyMigrateFilesTask, group: roxyGroup, description: "Migrate Roxy source files into this Gradle project. " + + "Use -ProxyProjectPath to define the location of your Roxy project.") + project.task("mlRoxyMigrateProperties", type: RoxyMigratePropertiesTask, group: roxyGroup, description: "Migrate Roxy properties into the gradle.properties file in this project. " + + "Use -ProxyProjectPath to define the location of your Roxy project.") + project.task("mlRoxyMigrateProject", group: roxyGroup, description: "Run all tasks for migrating a Roxy project into this Gradle project. " + + "Use -ProxyProjectPath to define the location of your Roxy project.", dependsOn: ["mlRoxyMigrateBuildSteps", "mlRoxyMigrateFiles", "mlRoxyMigrateProperties"]) logger.info("Finished initializing ml-gradle\n") } diff --git a/src/main/groovy/com/marklogic/gradle/task/MarkLogicTask.groovy b/src/main/groovy/com/marklogic/gradle/task/MarkLogicTask.groovy index cd8dccb6b..9c88b1db9 100644 --- a/src/main/groovy/com/marklogic/gradle/task/MarkLogicTask.groovy +++ b/src/main/groovy/com/marklogic/gradle/task/MarkLogicTask.groovy @@ -44,11 +44,7 @@ class MarkLogicTask extends DefaultTask { project.hasProperty("mlAdminPassword") ? project.property("mlAdminPassword") : project.property("mlPassword") } - String getRoxyHome(){ - project.hasProperty("mlRoxyHome") ? project.property("mlRoxyHome") : null - } - - DatabaseClient newClient() { + DatabaseClient newClient() { getAppConfig().newDatabaseClient() } diff --git a/src/main/groovy/com/marklogic/gradle/task/ServerEvalTask.groovy b/src/main/groovy/com/marklogic/gradle/task/ServerEvalTask.groovy new file mode 100644 index 000000000..4eb29c44b --- /dev/null +++ b/src/main/groovy/com/marklogic/gradle/task/ServerEvalTask.groovy @@ -0,0 +1,28 @@ +package com.marklogic.gradle.task + +import com.marklogic.client.DatabaseClient +import org.gradle.api.tasks.TaskAction + +class ServerEvalTask extends MarkLogicTask { + + String xquery + String javascript + + @TaskAction + void serverEval() { + DatabaseClient client = newClient() + try { + String result + if (xquery != null) { + result = client.newServerEval().xquery(xquery).evalAs(String.class) + } else { + result = client.newServerEval().javascript(javascript).evalAs(String.class) + } + if (result != null) { + println result + } + } finally { + client.release() + } + } +} diff --git a/src/main/groovy/com/marklogic/gradle/task/roxy/RoxyCopyPropertiesTask.groovy b/src/main/groovy/com/marklogic/gradle/task/roxy/RoxyCopyPropertiesTask.groovy deleted file mode 100644 index 152512e6f..000000000 --- a/src/main/groovy/com/marklogic/gradle/task/roxy/RoxyCopyPropertiesTask.groovy +++ /dev/null @@ -1,52 +0,0 @@ -package com.marklogic.gradle.task.roxy - -import com.marklogic.gradle.task.MarkLogicTask -import org.gradle.api.tasks.TaskAction - -class RoxyCopyPropertiesTask extends MarkLogicTask { - - def roxyPropertyFiles = ["default.properties", "build.properties"] - def roxyGradleMapping = ["app-name" : "mlAppName", "app-port" : "mlRestPort", - "user" : "mlRestAdminUsername", "password" : "mlAdminPassword", - "content-forests-per-host" : "mlContentForestsPerHost", "group" : "mlGroupName", - "forest-data-dir": "envForestDataDirectory"] - - @TaskAction - void copyProperties() { - if (null != getRoxyHome()) { - Map roxyProperties = new LinkedHashMap() - roxyPropertyFiles.each { propertyFile -> - new File(getRoxyHome() + "/deploy/", propertyFile).eachLine { line -> - if (!line.startsWith("#")) { - def keyValue = line.split("=") - if(keyValue.length == 2) - roxyProperties.put(keyValue[0], keyValue[1]) - } - } - } - writeFile("gradle.properties", constructText(roxyProperties)) - }else{ - println "mlRoxyHome parameter is not provided. Please run using -P mlRoxyHome=/your/project/home"; - } - } - - String constructText(Map roxyProperties){ - def properties = new StringBuilder() - roxyGradleMapping.each { entry -> - def val = roxyProperties.get(entry.key) - if(null != val) - properties.append(entry.value).append("=").append(val).append("\n") - } - return properties - } - - void writeFile(String filename, String text) { - File file = new File(filename); - if (file.exists()) { - new File("backup-" + filename).write(file.text) - } - println "Writing: " + filename - file.write(text) - } - -} diff --git a/src/main/groovy/com/marklogic/gradle/task/roxy/RoxyMigrateBuildStepsTask.groovy b/src/main/groovy/com/marklogic/gradle/task/roxy/RoxyMigrateBuildStepsTask.groovy new file mode 100644 index 000000000..b7fa07b45 --- /dev/null +++ b/src/main/groovy/com/marklogic/gradle/task/roxy/RoxyMigrateBuildStepsTask.groovy @@ -0,0 +1,183 @@ +package com.marklogic.gradle.task.roxy + +import org.gradle.api.tasks.TaskAction + +/** + * Does some hacky string parsing to try to convert custom build steps in the Roxy app_specific.rb file into custom + * Gradle tasks. + */ +class RoxyMigrateBuildStepsTask extends RoxyTask { + + @TaskAction + void copyBuildSteps() { + if (getRoxyProjectPath()) { + def appSpecificPath = project.hasProperty("appSpecificPath") ? project.property("appSpecificPath") : "deploy/app_specific.rb" + def filePath = getRoxyProjectPath() + "/" + appSpecificPath + if (new File(filePath).exists()) { + copyBuildStepsToGradleFile(filePath) + } else { + println "Not copying build steps, did not find Roxy app-specific build file at: " + filePath + } + } else { + printMissingPathMessage() + } + } + + void copyBuildStepsToGradleFile(filePath) { + println "Copying build steps from: " + filePath + + List buildSteps = extractBuildSteps(filePath) + int count = buildSteps.size() + + if (count == 0) { + println "No custom build steps were found in: " + filePath + } + else { + File file = new File("build.gradle"); + println "Backing up build.gradle to backup-build.gradle" + new File("backup-build.gradle").write(file.text) + + String newText = file.text + "\n" + for (BuildStep step : buildSteps) { + if (!step.bodyLines.isEmpty()) { + println "Copying Roxy buld step: " + step.name + newText += "\n" + step.toGradleTask() + } else { + println "Did not copy Roxy build step, could not extract method body: " + step.name + } + } + + String message = "Adding " + count; + if (count > 1) { + message += " tasks" + } else { + message += " task" + } + println message + " to build.gradle" + file.write(newText) + } + } + + List extractBuildSteps(String filePath) { + String text = new File(filePath).text + + String[] lines = text.split("\\n"); + List buildSteps = new ArrayList<>(); + + BuildStep currentBuildStep = null; + + for (int i = 0; i < lines.length; i++) { + String line = lines[i]; + if (line.trim().startsWith("#")) { + // Comment line, ignore + continue; + } + else if (line.trim().startsWith("def ") && line.contains("()")) { + // We're starting a new build step + currentBuildStep = new BuildStep(); + buildSteps.add(currentBuildStep); + currentBuildStep.name = extractBuildStepName(line); + } + else if (currentBuildStep != null) { + if (line.startsWith("@logger")) { + String message = extractLogMessage(line); + if (message != null) { + currentBuildStep.logMessages.add(message); + } + } else if (line.contains("execute_query")) { + i++; + for (; i < lines.length; i++) { + line = lines[i]; + if (line.trim().equals("},") || line.trim().equals("}")) { + break; + } else { + currentBuildStep.bodyLines.add(line); + } + } + } + } + } + + return buildSteps + } + + /** + * Assumes e.g. "def my_method-name()". + * + * @param line + * @return + */ + String extractBuildStepName(String line) { + int pos = line.indexOf("def "); + int end = line.indexOf("("); + return line.substring(pos + 4, end); + } + + /** + * We ignore the log message if it appears to do string concatenation, as odds are we can't resolve the + * variables. + * + * Assumes e.g. @logger.info "Hello world" + * + * @param line + * @return + */ + String extractLogMessage(String line) { + if (line.contains("+")) { + return null; + } + int start = line.indexOf("\""); + if (start > -1) { + int end = start + 1 + line.substring(start + 1).indexOf("\""); + if (end > start && end < line.length()) { + return line.substring(start + 1, end); + } + } + return null; + } + + /** + * Used for backing up and then overwriting the Gradle build file. + * + * @param filename + * @param text + */ + void writeFile(String filename, String text) { + File file = new File(filename); + if (file.exists()) { + new File("backup-" + filename).write(file.text) + } + println "Writing: " + filename + file.write(text) + } +} + +class BuildStep { + + String name; + List bodyLines = new ArrayList<>(); + List logMessages = new ArrayList<>(); + + String toGradleTask() { + StringBuilder sb = new StringBuilder("task " + name + "(type: com.marklogic.gradle.task.ServerEvalTask, group: 'Converted Roxy build step') {\n"); + if (!logMessages.isEmpty()) { + sb.append("\tdoFirst {\n"); + for (String message : logMessages) { + sb.append("\t\tprintln \"" + message + "\""); + } + sb.append("\t}\n"); + } + sb.append("\txquery = "); + int lineCount = bodyLines.size(); + for (int i = 0; i < lineCount; i++) { + String line = bodyLines.get(i).replaceAll("'", "\""); + sb.append("'" + line + "'"); + if (i < lineCount - 1) { + sb.append(" +"); + } + sb.append("\n"); + } + sb.append("}\n"); + return sb.toString(); + } +} diff --git a/src/main/groovy/com/marklogic/gradle/task/roxy/RoxyCopyFilesTask.groovy b/src/main/groovy/com/marklogic/gradle/task/roxy/RoxyMigrateFilesTask.groovy similarity index 54% rename from src/main/groovy/com/marklogic/gradle/task/roxy/RoxyCopyFilesTask.groovy rename to src/main/groovy/com/marklogic/gradle/task/roxy/RoxyMigrateFilesTask.groovy index 6b609141a..2d14379a0 100644 --- a/src/main/groovy/com/marklogic/gradle/task/roxy/RoxyCopyFilesTask.groovy +++ b/src/main/groovy/com/marklogic/gradle/task/roxy/RoxyMigrateFilesTask.groovy @@ -1,10 +1,9 @@ package com.marklogic.gradle.task.roxy -import com.marklogic.gradle.task.MarkLogicTask import org.apache.commons.io.FileUtils import org.gradle.api.tasks.TaskAction -class RoxyCopyFilesTask extends MarkLogicTask { +class RoxyMigrateFilesTask extends RoxyTask { def roxyFolderMapping = [ "src" : "/root", @@ -15,27 +14,25 @@ class RoxyCopyFilesTask extends MarkLogicTask { @TaskAction void copyRoxyFiles() { - if (getRoxyHome()) { + if (getRoxyProjectPath()) { def baseDir = getAppConfig().getModulePaths().get(0) roxyFolderMapping.each { k, v -> - def source = getRoxyHome() + "/" + k - println "Source folder '" + source + "' ... " - def sourceFolder = new File(source) + def sourcePath = getRoxyProjectPath() + "/" + k + def sourceFolder = new File(sourcePath) if (sourceFolder.exists() && sourceFolder.isDirectory()) { def targetDir = baseDir + v - println "Creating folder '" + targetDir + "' ... " + println "Creating directory: " + targetDir def targetFolder = new File(targetDir) FileUtils.forceMkdir(targetFolder) - println "Copying contents of '" + source + "' to '" + targetDir + "' ... " + println "Copying contents of '" + sourcePath + "' to '" + targetDir + "'" FileUtils.copyDirectory(sourceFolder, targetFolder) + } else { + println "Did not find Roxy source directory: " + sourcePath } } } else { - println "mlRoxyHome parameter is not provided. Please run using -PmlRoxyHome=/your/roxy/project/home" + printMissingPathMessage() } } - String getRoxyHome(){ - project.hasProperty("mlRoxyHome") ? project.property("mlRoxyHome") : "" - } } diff --git a/src/main/groovy/com/marklogic/gradle/task/roxy/RoxyMigratePropertiesTask.groovy b/src/main/groovy/com/marklogic/gradle/task/roxy/RoxyMigratePropertiesTask.groovy new file mode 100644 index 000000000..28b487e7e --- /dev/null +++ b/src/main/groovy/com/marklogic/gradle/task/roxy/RoxyMigratePropertiesTask.groovy @@ -0,0 +1,93 @@ +package com.marklogic.gradle.task.roxy + +import org.gradle.api.tasks.TaskAction + +class RoxyMigratePropertiesTask extends RoxyTask { + + def roxyPropertyFiles = ["default.properties", "build.properties"] + def roxyGradleMapping = [ + "app-name" : "mlAppName", + "app-port" : "mlRestPort", + "appuser-password" : "mlRestAdminPassword", + "authentication-method" : "mlRestAuthentication", + "content-db" : "mlContentDatabaseName", + "content-forests-per-host": "mlContentForestsPerHost", + "group" : "mlGroupName", + "modules-db" : "mlModulesDatabaseName", + "password" : "mlPassword", + "test-port" : "mlTestRestPort", + "user" : "mlUsername" + ] + + @TaskAction + void copyProperties() { + if (null != getRoxyProjectPath()) { + Map roxyProperties = buildMapOfRoxyProperties() + writeFile("gradle.properties", constructText(roxyProperties)) + } else { + printMissingPathMessage() + } + } + + Map buildMapOfRoxyProperties() { + Map roxyProperties = new LinkedHashMap() + roxyPropertyFiles.each { propertyFile -> + File file = new File(getRoxyProjectPath() + "/deploy/", propertyFile) + if (file.exists()) { + file.eachLine { line -> + if (!line.startsWith("#") && !line.isEmpty()) { + def keyValue = line.split("=") + if (keyValue.length == 2) { + roxyProperties.put(keyValue[0], keyValue[1]) + } + } + } + } else { + println "Did not find Roxy properties file: " + getRoxyProjectPath() + "/deploy/" + propertyFile + } + } + return roxyProperties + } + + /** + * First write the Roxy properties that can be mapped to ml-gradle properties. Then write every other Roxy property. + * + * @param roxyProperties + * @return + */ + String constructText(Map roxyProperties) { + def properties = new StringBuilder() + + properties.append("# Roxy properties that were mapped to ml-gradle properties\n") + roxyGradleMapping.each { entry -> + def val = roxyProperties.get(entry.key) + if (val != null) { + properties.append(entry.value).append("=").append(val).append("\n") + } + } + + properties.append("\n# All other Roxy properties\n") + roxyProperties.each { entry -> + String key = entry.key + if (roxyGradleMapping.containsKey(key)) { + key = roxyGradleMapping.get(key) + println "Mapping Roxy property '" + entry.key + "' to ml-gradle property '" + key + "'" + } + def val = roxyProperties.get(entry.key) + if (null != val) { + properties.append(key).append("=").append(val).append("\n") + } + } + return properties + } + + void writeFile(String filename, String text) { + File file = new File(filename); + if (file.exists()) { + new File("backup-" + filename).write(file.text) + } + println "Writing: " + filename + file.write(text) + } + +} diff --git a/src/main/groovy/com/marklogic/gradle/task/roxy/RoxyTask.groovy b/src/main/groovy/com/marklogic/gradle/task/roxy/RoxyTask.groovy new file mode 100644 index 000000000..cb7fa203a --- /dev/null +++ b/src/main/groovy/com/marklogic/gradle/task/roxy/RoxyTask.groovy @@ -0,0 +1,14 @@ +package com.marklogic.gradle.task.roxy + +import com.marklogic.gradle.task.MarkLogicTask + +class RoxyTask extends MarkLogicTask { + + String getRoxyProjectPath() { + project.hasProperty("roxyProjectPath") ? project.property("roxyProjectPath") : null + } + + void printMissingPathMessage() { + println "The 'roxyProjectPath' property is not defined. Please run using -ProxyProjectPath=/path/to/your/roxy/project" + } +} From a95c03d8a54b60496c4c71e0437d476679b59ee1 Mon Sep 17 00:00:00 2001 From: rjrudin Date: Mon, 24 Jul 2017 21:37:19 -0400 Subject: [PATCH 05/12] Updated README --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 81e95901a..fc2caf5de 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,11 @@ What is Gradle? =============== -Gradle is a standard build tool that is used for building and deploying primarily Java applications, but it can be used for any codebase. -The full user guide for Gradle is available at http://www.gradle.org/docs/current/userguide/userguide.html. +Gradle is a standard build tool that is used for building and deploying primarily Java applications, but it can be used for any codebase. Check out the [Gradle user guide](http://www.gradle.org/docs/current/userguide/userguide.html) for more information. What is ml-gradle? ========= -ml-gradle is a [Gradle plugin](http://www.gradle.org/plugins "") that supports a number of tasks pertaining to deploying an +ml-gradle is a [Gradle plugin](https://plugins.gradle.org/ "") that supports a number of tasks pertaining to deploying an application to MarkLogic and interacting with other features of MarkLogic via a Gradle build file. The bulk of the functionality provided by ml-gradle is actually in [ml-app-deployer](https://github.com/rjrudin/ml-app-deployer) - ml-gradle is just intended to be a thin wrapper around this library, exposing its functionality via Gradle tasks and properties. From 824c43b4ad682ec751714584fb5fb6cc5c66efc7 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Thu, 27 Jul 2017 10:40:21 -0400 Subject: [PATCH 06/12] #222 Commands can now be ignored when running mlDeploy --- .../local-testing-project/gradle.properties | 2 +- .../marklogic/gradle/MarkLogicPlugin.groovy | 3 ++- .../gradle/task/DeployAppTask.groovy | 25 ++++++++++++++++++- .../task/export/ExportResourcesTask.groovy | 4 +-- 4 files changed, 29 insertions(+), 5 deletions(-) diff --git a/examples/local-testing-project/gradle.properties b/examples/local-testing-project/gradle.properties index a45c7e42b..9d2718363 100644 --- a/examples/local-testing-project/gradle.properties +++ b/examples/local-testing-project/gradle.properties @@ -1,6 +1,6 @@ # Set this to the version you used when running # "gradle -Pversion=(something) publishToMavenLocal" on your local ml-gradle repo -mlGradleVersion=2.8.0 +mlGradleVersion=2.9.0 mlHost=localhost mlAppName=example diff --git a/src/main/groovy/com/marklogic/gradle/MarkLogicPlugin.groovy b/src/main/groovy/com/marklogic/gradle/MarkLogicPlugin.groovy index 838bfdd63..8d91f593d 100644 --- a/src/main/groovy/com/marklogic/gradle/MarkLogicPlugin.groovy +++ b/src/main/groovy/com/marklogic/gradle/MarkLogicPlugin.groovy @@ -83,7 +83,8 @@ class MarkLogicPlugin implements Plugin { String deployGroup = "ml-gradle Deploy" project.task("mlPostDeploy", group: deployGroup, description: "Add dependsOn to this task to add tasks at the end of mlDeploy").mustRunAfter(["mlDeployApp"]) project.task("mlPostUndeploy", group: deployGroup, description: "Add dependsOn to this task to add tasks at the end of mlUndeploy").mustRunAfter(["mlUndeployApp"]) - project.task("mlDeploy", group: deployGroup, dependsOn: ["mlDeployApp", "mlPostDeploy"], description: "Deploys all application resources in the configuration directory and allows for additional steps via mlPostDeploy.dependsOn").mustRunAfter("mlClearModulesDatabase") + project.task("mlDeploy", group: deployGroup, dependsOn: ["mlDeployApp", "mlPostDeploy"], + description: "Deploys all application resources in the configuration directory and allows for additional steps via mlPostDeploy.dependsOn. Use -Pignore to specify a comma-delimited list of short class names of ml-app-deployer command classes to ignore while deploying.").mustRunAfter("mlClearModulesDatabase") project.task("mlUndeploy", group: deployGroup, dependsOn: ["mlUndeployApp", "mlPostUndeploy"], description: "Undeploys all application resources in the configuration directory and allows for additional steps via mlPostUndeploy.dependsOn") project.task("mlRedeploy", group: deployGroup, dependsOn: ["mlClearModulesDatabase", "mlDeploy"], description: "Clears the modules database and then deploys the application") diff --git a/src/main/groovy/com/marklogic/gradle/task/DeployAppTask.groovy b/src/main/groovy/com/marklogic/gradle/task/DeployAppTask.groovy index 174bb444a..ba412280a 100644 --- a/src/main/groovy/com/marklogic/gradle/task/DeployAppTask.groovy +++ b/src/main/groovy/com/marklogic/gradle/task/DeployAppTask.groovy @@ -1,11 +1,34 @@ package com.marklogic.gradle.task +import com.marklogic.appdeployer.impl.SimpleAppDeployer import org.gradle.api.tasks.TaskAction class DeployAppTask extends MarkLogicTask { + /** + * Use "-Pignore" to specify the short class names of ml-app-deployer commands to ignore. The commands are then + * removed from the mlAppDeployer object, as long as it is an instance of SimpleAppDeployer from ml-app-deployer. + */ @TaskAction void deployApp() { - getAppDeployer().deploy(getAppConfig()) + def appDeployer = getAppDeployer() + if (project.hasProperty("ignore")) { + if (appDeployer instanceof SimpleAppDeployer) { + String[] commandNames = project.property("ignore").split(",") + SimpleAppDeployer deployer = (SimpleAppDeployer)appDeployer + for (String commandName : commandNames) { + def command = deployer.removeCommand(commandName) + if (command != null) { + println "Ignoring command: " + commandName + } else { + println "Could not find command specified by ignore property: " + commandName + } + } + } + else { + println "ignore property defined, but mlAppDeployer is not an instance of SimpleAppDeployer, so not able to ignore commands" + } + } + appDeployer.deploy(getAppConfig()) } } diff --git a/src/main/groovy/com/marklogic/gradle/task/export/ExportResourcesTask.groovy b/src/main/groovy/com/marklogic/gradle/task/export/ExportResourcesTask.groovy index 97f57fffa..f66a15dd6 100644 --- a/src/main/groovy/com/marklogic/gradle/task/export/ExportResourcesTask.groovy +++ b/src/main/groovy/com/marklogic/gradle/task/export/ExportResourcesTask.groovy @@ -4,7 +4,7 @@ import com.marklogic.appdeployer.export.ExportedResources import com.marklogic.appdeployer.export.Exporter import com.marklogic.gradle.task.MarkLogicTask import com.marklogic.mgmt.selector.PrefixResourceSelector -import com.marklogic.mgmt.selector.PropertiesFileResourceSelector +import com.marklogic.mgmt.selector.PropertiesResourceSelector import com.marklogic.mgmt.selector.RegexResourceSelector import com.marklogic.mgmt.selector.ResourceSelector import org.gradle.api.tasks.TaskAction @@ -27,7 +27,7 @@ class ExportResourcesTask extends MarkLogicTask { String filename = getProject().property(filePropName) File file = new File(filename) if (file.exists()) { - export(new PropertiesFileResourceSelector(file)) + export(new PropertiesResourceSelector(file)) } else { println "File " + filename + " does not exist" } From 45209b0a99503bb2fc95409178b004820914df9c Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Wed, 2 Aug 2017 10:08:49 -0400 Subject: [PATCH 07/12] Updating docs for failover project example --- examples/failover-project/README.md | 7 +++++-- examples/failover-project/gradle.properties | 14 +++++++++----- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/examples/failover-project/README.md b/examples/failover-project/README.md index 640131f72..f200724b7 100644 --- a/examples/failover-project/README.md +++ b/examples/failover-project/README.md @@ -1,6 +1,9 @@ This project shows an example of configuring replica forests for the out-of-the-box databases that are typically worth -enabling failover. +enabling failover. See https://github.com/marklogic-community/ml-gradle/wiki/Property-reference for more properties +for configuring the replica forests that are created. Note that you'll need to run this against a cluster with at least two nodes (failover requires three nodes, but you only need two nodes to see how ml-gradle will create replica forests). - \ No newline at end of file + +For complete control of primary and replica forests, check out +the [custom forests and replicas](../custom-forests-and-replicas-project) project. diff --git a/examples/failover-project/gradle.properties b/examples/failover-project/gradle.properties index 2810dd028..9c06b33d1 100644 --- a/examples/failover-project/gradle.properties +++ b/examples/failover-project/gradle.properties @@ -4,9 +4,13 @@ mlRestPort=8013 mlUsername=admin mlPassword=admin -# The following configuration property lets you easily define how many replicas you want of the forests -# for each database. For more precise control over forests and replicas, look into DeployCustomForestsCommand -# and DeployCustomForestsTask. -#mlDatabaseNamesAndReplicaCounts=Security,2,Schemas,1,Meters,1,App-Services,1 +# mlDatabaseNamesAndReplicaCounts lets you easily configure the number of replica forests you want for each primary +# forest attached to a given database. + +# This example shows configuring multiple default databases with replicas. +# mlDatabaseNamesAndReplicaCounts=Security,2,Schemas,1,Meters,1,App-Services,1 + +# This example shows configuring the database created by this project with replicas. You can of course configure the +# number of primary forests you want for the content database too. mlContentForestsPerHost=1 -mlDatabaseNamesAndReplicaCounts=failover-example-content,1 \ No newline at end of file +mlDatabaseNamesAndReplicaCounts=failover-example-content,1 From 2854395b6cff88892707325602ac3bb8774c2d72 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Mon, 7 Aug 2017 09:15:39 -0400 Subject: [PATCH 08/12] Bumping to 2.9.0 --- build.gradle | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index 7ea7766db..7e384ccb1 100644 --- a/build.gradle +++ b/build.gradle @@ -90,11 +90,11 @@ pluginBundle { displayName = 'ml-gradle for MarkLogic' description = 'Gradle plugin for configuring and deploying applications to MarkLogic' tags = ['marklogic'] - version = "2.8.0" + version = "2.9.0" } } mavenCoordinates { - version = "2.8.0" + version = "2.9.0" } -} \ No newline at end of file +} From b1e6782c70661126fb982b376bd17ee5d0bf46dc Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Mon, 7 Aug 2017 09:23:16 -0400 Subject: [PATCH 09/12] #214 Added docs about role ordering --- examples/role-project/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/examples/role-project/README.md b/examples/role-project/README.md index 001f939fb..e86956d51 100644 --- a/examples/role-project/README.md +++ b/examples/role-project/README.md @@ -1,2 +1,8 @@ This project shows how to create a role that refers to itself. See the src/main/ml-config/security/roles directory for more information. + +As of version 2.9.0, you don't need to do anything special now. ml-gradle will detect that a role refers to itself, +and it will deploy the role twice - once without its role dependencies, and again with its role dependencies. + +In version 2.9.0, ml-gradle will also figure out what order to deploy roles in - no need to order them yourself via +their filenames. From 9f9b4b5e1358c2fad1b1f0a48fc14ad641e254c9 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Mon, 7 Aug 2017 17:32:56 -0400 Subject: [PATCH 10/12] Slimming down README --- README.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/README.md b/README.md index fc2caf5de..251ba4264 100644 --- a/README.md +++ b/README.md @@ -45,9 +45,3 @@ how different resources can be configured. Finally, if you run into trouble, you can either submit an issue to this project or try asking a question [on stackoverflow with marklogic as a tag](http://stackoverflow.com/questions/tagged/marklogic). - -Can I use ml-gradle with Roxy? -========= -You bet! Starting with version 2.2.0, you can use "gradle mlWatch" to automatically load new/modified modules into the -modules database of your Roxy application, and ml-gradle will substitute tokens based on Roxy properties as well. -See [Loading modules in a Roxy project](https://github.com/rjrudin/ml-gradle/wiki/Loading-modules-on-a-Roxy-project) for more information. From 6e755b0679a95f34eeba094aef03e22b621ea0d8 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Thu, 7 Sep 2017 09:18:11 -0400 Subject: [PATCH 11/12] sample-project now uses 3.0.0 --- examples/sample-project/build.gradle | 4 ++-- .../src/test/java/sample/GetClientConfigTest.java | 2 +- .../src/test/java/sample/SampleProjectTestConfig.java | 2 +- .../test/java/sample/SearchDocumentsViaRestAssuredTest.java | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/sample-project/build.gradle b/examples/sample-project/build.gradle index ace47e937..d5c86c7d5 100644 --- a/examples/sample-project/build.gradle +++ b/examples/sample-project/build.gradle @@ -8,7 +8,7 @@ buildscript { jcenter() } dependencies { - classpath "com.marklogic:ml-gradle:3.0-alpha1" + classpath "com.marklogic:ml-gradle:3.0.0" } } @@ -53,7 +53,7 @@ configurations { */ dependencies { // Needed to compile and run the JUnit tests - testCompile "com.marklogic:ml-junit:3.0-alpha1" + testCompile "com.marklogic:ml-junit:3.0.0" testCompile "com.jayway.restassured:rest-assured:2.4.1" // corb jar available from jcenter diff --git a/examples/sample-project/src/test/java/sample/GetClientConfigTest.java b/examples/sample-project/src/test/java/sample/GetClientConfigTest.java index ab4789d3e..67f3afc11 100644 --- a/examples/sample-project/src/test/java/sample/GetClientConfigTest.java +++ b/examples/sample-project/src/test/java/sample/GetClientConfigTest.java @@ -2,7 +2,7 @@ import org.junit.Test; -import com.marklogic.client.ext.helper.DatabaseClientConfig; +import com.marklogic.client.ext.DatabaseClientConfig; public class GetClientConfigTest extends AbstractSampleProjectTest { diff --git a/examples/sample-project/src/test/java/sample/SampleProjectTestConfig.java b/examples/sample-project/src/test/java/sample/SampleProjectTestConfig.java index b1033884d..05eaf0f95 100644 --- a/examples/sample-project/src/test/java/sample/SampleProjectTestConfig.java +++ b/examples/sample-project/src/test/java/sample/SampleProjectTestConfig.java @@ -4,7 +4,7 @@ import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.PropertySource; -import com.marklogic.client.ext.helper.DatabaseClientConfig; +import com.marklogic.client.ext.DatabaseClientConfig; import com.marklogic.junit.spring.BasicTestConfig; @Configuration diff --git a/examples/sample-project/src/test/java/sample/SearchDocumentsViaRestAssuredTest.java b/examples/sample-project/src/test/java/sample/SearchDocumentsViaRestAssuredTest.java index 8fc47c44b..a904bfb85 100644 --- a/examples/sample-project/src/test/java/sample/SearchDocumentsViaRestAssuredTest.java +++ b/examples/sample-project/src/test/java/sample/SearchDocumentsViaRestAssuredTest.java @@ -9,7 +9,7 @@ import com.jayway.restassured.RestAssured; import com.jayway.restassured.path.json.JsonPath; import com.marklogic.client.document.XMLDocumentManager; -import com.marklogic.client.ext.helper.DatabaseClientConfig; +import com.marklogic.client.ext.DatabaseClientConfig; import com.marklogic.client.io.DocumentMetadataHandle; import com.marklogic.client.io.Format; import com.marklogic.client.io.StringHandle; From fbca0ff148cbbff288b5b16818fe73ce21abdce3 Mon Sep 17 00:00:00 2001 From: Paxton Hare Date: Fri, 22 Sep 2017 12:51:05 -0400 Subject: [PATCH 12/12] updated to use marklogic-client-api 4.0.2 added a sample spock test and travis configs --- .travis.yml | 19 ++++ build.gradle | 12 ++- shared/dev-tasks/install-dependencies.sh | 6 ++ shared/dev-tasks/run-tests.sh | 6 ++ shared/dev-tasks/setup-marklogic.sh | 12 +++ shared/dev-tasks/travis-install-ml.sh | 77 ++++++++++++++++ .../marklogic/gradle/MarkLogicPlugin.groovy | 18 ---- .../com/marklogic/gradle/BaseTest.groovy | 87 +++++++++++++++++++ .../marklogic/gradle/DeployAppTaskTest.groovy | 37 ++++++++ 9 files changed, 254 insertions(+), 20 deletions(-) create mode 100644 .travis.yml create mode 100644 shared/dev-tasks/install-dependencies.sh create mode 100644 shared/dev-tasks/run-tests.sh create mode 100644 shared/dev-tasks/setup-marklogic.sh create mode 100755 shared/dev-tasks/travis-install-ml.sh create mode 100644 src/test/groovy/com/marklogic/gradle/BaseTest.groovy create mode 100644 src/test/groovy/com/marklogic/gradle/DeployAppTaskTest.groovy diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000..d9cea7813 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,19 @@ +language: java +jdk: + - oraclejdk8 +sudo: true +before_install: + - echo 'America/Los_Angeles' | sudo tee /etc/timezone + - sudo dpkg-reconfigure --frontend noninteractive tzdata + - chmod 755 ./shared/dev-tasks/*.sh + - chmod 755 ./gradlew +install: + - ./shared/dev-tasks/install-dependencies.sh +script: + - ./shared/dev-tasks/run-tests.sh + +#whitelist +branches: + only: + - master + - dev diff --git a/build.gradle b/build.gradle index 7fbf2059b..5f818ca94 100644 --- a/build.gradle +++ b/build.gradle @@ -6,10 +6,11 @@ plugins { id "com.jfrog.bintray" version "1.6" id "com.github.jk1.dependency-license-report" version "0.3.11" id "com.gradle.plugin-publish" version "0.9.7" + id "java-gradle-plugin" } -sourceCompatibility = "1.7" -targetCompatibility = "1.7" +sourceCompatibility = "1.8" +targetCompatibility = "1.8" repositories { mavenLocal() // Used for local development only @@ -22,6 +23,13 @@ dependencies { compile mlAppDeployerDependency compile mlcpUtilDependency compile group: 'commons-io', name: 'commons-io', version: '2.5' + + testCompile localGroovy() + testCompile gradleTestKit() + testCompile 'xmlunit:xmlunit:1.3' + testCompile('org.spockframework:spock-core:1.1-groovy-2.4') { + exclude module: 'groovy-all' + } } task sourcesJar(type: Jar, dependsOn: classes) { diff --git a/shared/dev-tasks/install-dependencies.sh b/shared/dev-tasks/install-dependencies.sh new file mode 100644 index 000000000..5fd303395 --- /dev/null +++ b/shared/dev-tasks/install-dependencies.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +if [ "${TRAVIS_SECURE_ENV_VARS}" = "true" ] ; then + ./shared/dev-tasks/travis-install-ml.sh release + ./shared/dev-tasks/setup-marklogic.sh +fi diff --git a/shared/dev-tasks/run-tests.sh b/shared/dev-tasks/run-tests.sh new file mode 100644 index 000000000..92046a121 --- /dev/null +++ b/shared/dev-tasks/run-tests.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +if [ "${TRAVIS_SECURE_ENV_VARS}" = "true" ] ; then + cd ${TRAVIS_BUILD_DIR} + ./gradlew test -i +fi diff --git a/shared/dev-tasks/setup-marklogic.sh b/shared/dev-tasks/setup-marklogic.sh new file mode 100644 index 000000000..71f28c339 --- /dev/null +++ b/shared/dev-tasks/setup-marklogic.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +sudo /etc/init.d/MarkLogic start +sleep 10 +curl -X POST -d "" http://localhost:8001/admin/v1/init +sleep 10 +curl -X POST -H "Content-type: application/x-www-form-urlencoded" \ + --data "admin-username=admin" \ + --data "admin-password=admin" \ + --data "realm=public" \ + "http://localhost:8001/admin/v1/instance-admin" +sleep 10 diff --git a/shared/dev-tasks/travis-install-ml.sh b/shared/dev-tasks/travis-install-ml.sh new file mode 100755 index 000000000..46f0e79fb --- /dev/null +++ b/shared/dev-tasks/travis-install-ml.sh @@ -0,0 +1,77 @@ +#!/bin/bash + +# runs command from parameters and exits with the eoror code of the command +# if it fails +function successOrExit { + "$@" + local status=$? + if [ $status -ne 0 ]; then + echo "$1 exited with error: $status" + exit $status + fi +} + +set | grep TRAVIS + +test $1 && arg1=$1 +if [[ $arg1 = 'release' ]]; then + ver=${ML_VERSION} + fname=MarkLogic-${ver}.x86_64.rpm + fnamedeb="marklogic_" + fnamedeb=$fnamedeb$ver + suff="_amd64.deb" + fnamedeb=$fnamedeb$suff + + curl -c cookies.txt --data "email=${MLBUILD_USER}&password=${MLBUILD_PASSWORD}" https://developer.marklogic.com/login + dl_link=$(curl -b cookies.txt --data "download=/download/binaries/9.0/${fname}" https://developer.marklogic.com/get-download-url | perl -pe 's/.*"path":"([^"]+).*/\1/') + url="https://developer.marklogic.com${dl_link}" + + echo "********* Downloading MarkLogic $ver" + + successOrExit curl -k -o ./$fname $url + + fname=$(pwd)/$fname + + sudo apt-get update + sudo apt-get install wajig alien rpm lsb-base dpkg-dev debhelper build-essential + (cd /etc && sudo ln -s default sysconfig) + sudo wajig rpminstall $fname + + echo "********* MarkLogic $ver installed" +else + # find today + day=$(date +"%Y%m%d") + + # if the user passed a day string as a param then use it instead + test $1 && day=$1 + # make a version number out of the date + ver="9.0-$day" + + echo "********* Downloading MarkLogic nightly $ver" + + # fetch/install ML nightly + fname="MarkLogic-$ver.x86_64.rpm" + fnamedeb="marklogic_" + fnamedeb=$fnamedeb$ver + suff="_amd64.deb" + fnamedeb=$fnamedeb$suff + + url="https://root.marklogic.com/nightly/builds/linux64-rh7/rh7v-intel64-90-test-build.marklogic.com/b9_0/pkgs.$day/$fname" + + status=$(curl -k --anyauth -u $MLBUILD_USER:$MLBUILD_PASSWORD --head --write-out %{http_code} --silent --output /dev/null $url) + if [[ $status = 200 ]]; then + successOrExit curl -k --anyauth -u $MLBUILD_USER:$MLBUILD_PASSWORD -o ./$fname $url + + fname=$(pwd)/$fname + + sudo apt-get update + sudo apt-get install alien dpkg-dev debhelper build-essential + sudo alien -d -k $fname + sudo dpkg -i $fnamedeb + + echo "********* MarkLogic nightly $ver installed" + else + echo "CANNOT DOWNLOAD: status = $status for date $day (URL=\"$url\")" + exit 1 + fi +fi diff --git a/src/main/groovy/com/marklogic/gradle/MarkLogicPlugin.groovy b/src/main/groovy/com/marklogic/gradle/MarkLogicPlugin.groovy index 22b575e12..8f1e2e420 100644 --- a/src/main/groovy/com/marklogic/gradle/MarkLogicPlugin.groovy +++ b/src/main/groovy/com/marklogic/gradle/MarkLogicPlugin.groovy @@ -62,14 +62,10 @@ import com.marklogic.mgmt.ManageConfig import com.marklogic.mgmt.admin.AdminConfig import com.marklogic.mgmt.admin.AdminManager import com.marklogic.mgmt.admin.DefaultAdminConfigFactory -import com.sun.jersey.core.spi.component.ProviderServices import org.gradle.api.Plugin import org.gradle.api.Project import org.slf4j.LoggerFactory -import java.util.logging.Level -import java.util.logging.Logger - class MarkLogicPlugin implements Plugin { org.slf4j.Logger logger = LoggerFactory.getLogger(getClass()) @@ -77,8 +73,6 @@ class MarkLogicPlugin implements Plugin { void apply(Project project) { logger.info("\nInitializing ml-gradle") - quietDownJerseyLogging() - initializeAppDeployerObjects(project) project.getConfigurations().create("mlRestApi") @@ -293,16 +287,4 @@ class MarkLogicPlugin implements Plugin { deployer.setCommands(commands) return deployer } - - /** - * When the MarkLogic DatabaseClient class is used in Gradle, the Jersey ProviderServices class spits out - * a lot of not helpful logging at the INFO level. So we bump it down to WARNING to avoid that. - */ - void quietDownJerseyLogging() { - try { - Logger.getLogger(ProviderServices.class.getName()).setLevel(Level.WARNING) - } catch (Exception e) { - // Ignore, not important - } - } } diff --git a/src/test/groovy/com/marklogic/gradle/BaseTest.groovy b/src/test/groovy/com/marklogic/gradle/BaseTest.groovy new file mode 100644 index 000000000..8d33a5f92 --- /dev/null +++ b/src/test/groovy/com/marklogic/gradle/BaseTest.groovy @@ -0,0 +1,87 @@ +package com.marklogic.gradle + +import com.marklogic.mgmt.ManageClient +import com.marklogic.mgmt.ManageConfig +import com.marklogic.mgmt.admin.AdminConfig +import com.marklogic.mgmt.admin.AdminManager +import com.marklogic.mgmt.resource.appservers.ServerManager +import com.marklogic.mgmt.resource.databases.DatabaseManager +import com.marklogic.rest.util.ResourcesFragment +import org.custommonkey.xmlunit.XMLUnit +import org.gradle.testkit.runner.BuildResult +import org.gradle.testkit.runner.GradleRunner +import org.junit.rules.TemporaryFolder +import spock.lang.Specification + +class BaseTest extends Specification { + static final TemporaryFolder testProjectDir = new TemporaryFolder() + static File buildFile + static File propertiesFile + + static BuildResult runTask(String... task) { + return GradleRunner.create() + .withProjectDir(testProjectDir.root) + .withArguments(task) + .withDebug(true) + .withPluginClasspath() + .build() + } + + BuildResult runFailTask(String... task) { + return GradleRunner.create() + .withProjectDir(testProjectDir.root) + .withArguments(task) + .withDebug(true) + .withPluginClasspath().buildAndFail() + } + + static void createBuildFile() { + buildFile = testProjectDir.newFile('build.gradle') + buildFile << """ + plugins { + id 'com.marklogic.ml-gradle' + } + """ + } + + static AdminConfig getAdminConfig() { + return new AdminConfig("localhost", 8001, "admin", "admin") + } + + static AdminManager getAdminManager() { + return new AdminManager(getAdminConfig()) + } + + static ManageClient getManageClient() { + ManageConfig manageConfig = new ManageConfig("localhost", 8002, "admin", "admin") + ManageClient manageClient = new ManageClient(manageConfig) + return manageClient + } + + static ServerManager getServerManager() { + return new ServerManager(getManageClient()); + } + + static DatabaseManager getDatabaseManager() { + return new DatabaseManager(getManageClient()); + } + + static ResourcesFragment getDbConfig() { + return getDatabaseManager().getAsXml() + } + + static ResourcesFragment getServerConfig() { + return getServerManager().getAsXml() + } + + static void createPropertiesFile(contents) { + propertiesFile = testProjectDir.newFile('gradle.properties') + propertiesFile << contents + } + + def setupSpec() { + XMLUnit.setIgnoreWhitespace(true) + testProjectDir.create() + createBuildFile() + } +} diff --git a/src/test/groovy/com/marklogic/gradle/DeployAppTaskTest.groovy b/src/test/groovy/com/marklogic/gradle/DeployAppTaskTest.groovy new file mode 100644 index 000000000..e71457ae4 --- /dev/null +++ b/src/test/groovy/com/marklogic/gradle/DeployAppTaskTest.groovy @@ -0,0 +1,37 @@ +package com.marklogic.gradle + +import org.gradle.testkit.runner.UnexpectedBuildFailure + +import static org.gradle.testkit.runner.TaskOutcome.SUCCESS + +class DeployAppTaskTest extends BaseTest { + + def "deploy barebones app"() { + setup: + print(runTask('mlUnDeploy').output) + + expect: + def srf = getServerConfig() + def drf = getDbConfig() + !srf.resourceExists("my-app") + !drf.resourceExists("my-app-content") + !drf.resourceExists("my-app-modules") + + when: + def result = runTask('mlDeploy') + print(result.output) + + then: + notThrown(UnexpectedBuildFailure) + result.task(":mlDeploy").outcome == SUCCESS + def srf2 = getServerConfig() + def drf2 = getDbConfig() + srf2.resourceExists("my-app") + drf2.resourceExists("my-app-content") + drf2.resourceExists("my-app-modules") + + cleanup: + runTask('mlUndeploy') + } + +}