From fdd70e5edfb0743114a1825b025e3c0e1124d098 Mon Sep 17 00:00:00 2001 From: Stephen Mallette Date: Wed, 14 Sep 2016 11:35:56 -0400 Subject: [PATCH 1/8] Removed the second jython execution from gremlin-python build --- gremlin-python/pom.xml | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/gremlin-python/pom.xml b/gremlin-python/pom.xml index b1a9453d785..50c823da043 100644 --- a/gremlin-python/pom.xml +++ b/gremlin-python/pom.xml @@ -78,7 +78,7 @@ - + false ${maven.test.skip} ${project.parent.basedir}/gremlin-server @@ -285,13 +285,13 @@ + python profile if glVPython is present --> org.apache.maven.plugins maven-surefire-plugin - ${project.build.testOutputDirectory} + ${project.build.outputDirectory} **/jsr223/Python*Test.java @@ -320,7 +320,7 @@ maven-surefire-plugin - ${project.build.testOutputDirectory} + ${project.build.outputDirectory} @@ -331,19 +331,11 @@ 1.4 - package - - jython - - - - pythonDependencies - generate-test-resources + compile jython - ${project.build.testOutputDirectory} aenum==1.4.5 tornado==4.4.1 @@ -431,7 +423,7 @@ - + From e0d7c90d4391e24ae18fdb2cc6a57c978039b29f Mon Sep 17 00:00:00 2001 From: Stephen Mallette Date: Wed, 14 Sep 2016 11:59:07 -0400 Subject: [PATCH 2/8] Drop -DglvPython as a requirement for building gremlin-python. Now you can just add .glv file to the root of gremlin-python to let Maven know that your environment is python-enabled. --- .gitignore | 1 + .../dev/developer/development-environment.asciidoc | 11 ++++++++++- gremlin-python/pom.xml | 8 ++++---- pom.xml | 1 + 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 9d83c5a54fe..33fb2395125 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ benchmarks/ __pycache__/ *.py[cdo] __version__.py +.glv diff --git a/docs/src/dev/developer/development-environment.asciidoc b/docs/src/dev/developer/development-environment.asciidoc index 117f7925e70..632100c0db4 100644 --- a/docs/src/dev/developer/development-environment.asciidoc +++ b/docs/src/dev/developer/development-environment.asciidoc @@ -65,6 +65,16 @@ As of TinkerPop 3.2.2, the build optionally requires link:https://www.python.org Java tests that require Python code will be skipped. Developers should also install link:https://pypi.python.org/pypi/pip[pip] and link:https://virtualenv.pypa.io/en/stable/[virtualenv]. +Once the Python environment is established, the full building and testing of `gremlin-python` may commence. It can be +done manually from the command line with: + +[source,text] +mvn clean install -Pglv-python + +which enables the "glv-python" Maven profile or in a more automated fashion simply add a `.glv` file to the root of the +`gremlin-python` module which will signify to Maven that the environment is Python-ready. The `.glv` file need not have +any contents and is ignored by Git. A standard `mvn clean install` will then build `gremlin-python` in full. + [[release-environment]] Release Environment ~~~~~~~~~~~~~~~~~~~ @@ -117,7 +127,6 @@ mvn -Dmaven.javadoc.skip=true --projects tinkergraph-gremlin test ** Clean the `.groovy/grapes/org.apache.tinkerpop` directory on build: `mvn clean install -DcleanGrapes` ** Turn off "heavy" logging in the "process" tests: `mvn clean install -DargLine="-DmuteTestLogs=true"` ** The test suite for `neo4j-gremlin` is disabled by default - to turn it on: `mvn clean install -DincludeNeo4j` -** Build and execute with native Python tests (see <>: `mvn clean install -DglvPython` * Regenerate test data (only necessary given changes to IO classes): `mvn clean install -Dio` from `tinkergraph-gremlin` directory ** If there are changes to the Gryo format, it may be necessary to generate the Grateful Dead dataset from GraphSON (see `IoDataGenerationTest.shouldWriteGratefulDead`) * Check license headers are present: `mvn apache-rat:check` diff --git a/gremlin-python/pom.xml b/gremlin-python/pom.xml index 50c823da043..3695c0dba59 100644 --- a/gremlin-python/pom.xml +++ b/gremlin-python/pom.xml @@ -275,7 +275,7 @@ - glv-python-standard @@ -307,9 +307,9 @@ glv-python false - - glvPython - + + .glv + diff --git a/pom.xml b/pom.xml index fd620e743ef..e37746c4865 100644 --- a/pom.xml +++ b/pom.xml @@ -293,6 +293,7 @@ limitations under the License. **/src/main/static/** **/_bsp/** DEPENDENCIES + .glv From 2fbeb596d7fe72bccaf2d5a3e5eac0344882a3b1 Mon Sep 17 00:00:00 2001 From: Stephen Mallette Date: Wed, 14 Sep 2016 12:01:37 -0400 Subject: [PATCH 3/8] Updated changelog about -DglvPython --- CHANGELOG.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 24ed6010ccc..ab2f8df79ea 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -28,6 +28,7 @@ TinkerPop 3.2.3 (Release Date: NOT OFFICIALLY RELEASED YET) * Fixed a `JavaTranslator` bug where `Bytecode` instructions were being mutated during translation. * Added `Path` to Gremlin-Python with respective GraphSON 2.0 deserializer. +* New build options for `gremlin-python` where `-DglvPython` is no longer required. * Added missing `InetAddress` to GraphSON extension module. [[release-3-2-2]] From 2106b10e74008481b19bba98635f65dda99800bf Mon Sep 17 00:00:00 2001 From: Stephen Mallette Date: Wed, 14 Sep 2016 12:16:13 -0400 Subject: [PATCH 4/8] Allow -DskipTests to still produce a valid build of gremlin-python --- gremlin-python/pom.xml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/gremlin-python/pom.xml b/gremlin-python/pom.xml index 3695c0dba59..b2ba875be68 100644 --- a/gremlin-python/pom.xml +++ b/gremlin-python/pom.xml @@ -324,14 +324,13 @@ - net.sf.mavenjython jython-compile-maven-plugin 1.4 - compile + process-resources jython @@ -413,12 +412,11 @@ native-python-build - generate-test-resources + compile run - ${skipTests} From 9b4c7884dc5c3e09086113b3720f07e1501951c4 Mon Sep 17 00:00:00 2001 From: Stephen Mallette Date: Wed, 14 Sep 2016 13:45:41 -0400 Subject: [PATCH 5/8] Centrally manage gmavenplus plugin and bump to 1.5 --- giraph-gremlin/pom.xml | 18 ------------------ gremlin-console/pom.xml | 18 ------------------ gremlin-groovy-test/pom.xml | 18 ------------------ gremlin-groovy/pom.xml | 18 ------------------ gremlin-python/pom.xml | 22 ++-------------------- hadoop-gremlin/pom.xml | 18 ------------------ neo4j-gremlin/pom.xml | 18 ------------------ pom.xml | 25 ++++++++++++++++++++++++- spark-gremlin/pom.xml | 18 ------------------ 9 files changed, 26 insertions(+), 147 deletions(-) diff --git a/giraph-gremlin/pom.xml b/giraph-gremlin/pom.xml index 47b1adca4da..528b2ad8e14 100644 --- a/giraph-gremlin/pom.xml +++ b/giraph-gremlin/pom.xml @@ -228,24 +228,6 @@ limitations under the License. org.codehaus.gmavenplus gmavenplus-plugin - 1.2 - - - - addSources - addTestSources - generateStubs - compile - testGenerateStubs - testCompile - removeStubs - removeTestStubs - - - - - true - org.apache.maven.plugins diff --git a/gremlin-console/pom.xml b/gremlin-console/pom.xml index 7662a6b5c44..aae5e41b5ff 100644 --- a/gremlin-console/pom.xml +++ b/gremlin-console/pom.xml @@ -206,24 +206,6 @@ limitations under the License. org.codehaus.gmavenplus gmavenplus-plugin - 1.2 - - - - addSources - addTestSources - generateStubs - compile - testGenerateStubs - testCompile - removeStubs - removeTestStubs - - - true - - - diff --git a/gremlin-groovy-test/pom.xml b/gremlin-groovy-test/pom.xml index 2bb85026339..614591ed64f 100644 --- a/gremlin-groovy-test/pom.xml +++ b/gremlin-groovy-test/pom.xml @@ -50,24 +50,6 @@ limitations under the License. org.codehaus.gmavenplus gmavenplus-plugin - 1.2 - - - - addSources - addTestSources - generateStubs - compile - testGenerateStubs - testCompile - removeStubs - removeTestStubs - - - - - true - org.apache.maven.plugins diff --git a/gremlin-groovy/pom.xml b/gremlin-groovy/pom.xml index 5518be825b1..171770f3a56 100644 --- a/gremlin-groovy/pom.xml +++ b/gremlin-groovy/pom.xml @@ -113,24 +113,6 @@ limitations under the License. org.codehaus.gmavenplus gmavenplus-plugin - 1.2 - - - - addSources - addTestSources - generateStubs - compile - testGenerateStubs - testCompile - removeStubs - removeTestStubs - - - - - true - org.apache.maven.plugins diff --git a/gremlin-python/pom.xml b/gremlin-python/pom.xml index b2ba875be68..76afe6025b5 100644 --- a/gremlin-python/pom.xml +++ b/gremlin-python/pom.xml @@ -124,24 +124,6 @@ org.codehaus.gmavenplus gmavenplus-plugin - 1.2 - - - - addSources - addTestSources - generateStubs - compile - testGenerateStubs - testCompile - removeStubs - removeTestStubs - - - - - true - org.apache.maven.plugins @@ -458,7 +440,8 @@ native-python-test @@ -482,7 +465,6 @@ org.codehaus.gmavenplus gmavenplus-plugin - 1.2 org.codehaus.groovy diff --git a/hadoop-gremlin/pom.xml b/hadoop-gremlin/pom.xml index 542fd96a353..74f91b10757 100644 --- a/hadoop-gremlin/pom.xml +++ b/hadoop-gremlin/pom.xml @@ -223,24 +223,6 @@ limitations under the License. org.codehaus.gmavenplus gmavenplus-plugin - 1.2 - - - - addSources - addTestSources - generateStubs - compile - testGenerateStubs - testCompile - removeStubs - removeTestStubs - - - - - true - diff --git a/neo4j-gremlin/pom.xml b/neo4j-gremlin/pom.xml index d3e50822cfc..109f372db28 100644 --- a/neo4j-gremlin/pom.xml +++ b/neo4j-gremlin/pom.xml @@ -94,24 +94,6 @@ limitations under the License. org.codehaus.gmavenplus gmavenplus-plugin - 1.2 - - - - addSources - addTestSources - generateStubs - compile - testGenerateStubs - testCompile - removeStubs - removeTestStubs - - - - - true - org.apache.maven.plugins diff --git a/pom.xml b/pom.xml index e37746c4865..7a4c230b311 100644 --- a/pom.xml +++ b/pom.xml @@ -293,7 +293,7 @@ limitations under the License. **/src/main/static/** **/_bsp/** DEPENDENCIES - .glv + **/.glv @@ -433,6 +433,29 @@ limitations under the License. maven-javadoc-plugin ${javadoc-plugin.version} + + org.codehaus.gmavenplus + gmavenplus-plugin + 1.5 + + + + addSources + addTestSources + generateStubs + compile + testGenerateStubs + testCompile + removeStubs + removeTestStubs + + + + + 1.8 + true + + diff --git a/spark-gremlin/pom.xml b/spark-gremlin/pom.xml index e018b21111a..7f4137eee6e 100644 --- a/spark-gremlin/pom.xml +++ b/spark-gremlin/pom.xml @@ -362,24 +362,6 @@ org.codehaus.gmavenplus gmavenplus-plugin - 1.2 - - - - addSources - addTestSources - generateStubs - compile - testGenerateStubs - testCompile - removeStubs - removeTestStubs - - - - - true - org.apache.maven.plugins From 079cc20d9604c7bcbf21496a6fc042374f86b282 Mon Sep 17 00:00:00 2001 From: Stephen Mallette Date: Wed, 14 Sep 2016 14:14:20 -0400 Subject: [PATCH 6/8] Udpated dev docs related to release given changes to gremlin-python build --- .../developer/development-environment.asciidoc | 2 ++ docs/src/dev/developer/release.asciidoc | 16 ++++++++++------ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/docs/src/dev/developer/development-environment.asciidoc b/docs/src/dev/developer/development-environment.asciidoc index 632100c0db4..ade428823c5 100644 --- a/docs/src/dev/developer/development-environment.asciidoc +++ b/docs/src/dev/developer/development-environment.asciidoc @@ -14,6 +14,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. //// +[[development-environment]] Development Environment ======================= @@ -21,6 +22,7 @@ TinkerPop is fairly large body of code spread across many modules and covering m this complexity, it remains relatively straightforward a project to build. This following subsections explain how to configure a development environment for TinkerPop. +[[system-configuration]] System Configuration -------------------- diff --git a/docs/src/dev/developer/release.asciidoc b/docs/src/dev/developer/release.asciidoc index e1007e0ed78..f4d29ff55f9 100644 --- a/docs/src/dev/developer/release.asciidoc +++ b/docs/src/dev/developer/release.asciidoc @@ -32,6 +32,10 @@ from a previous version or from recent `SNAPSHOT`. When using one generated for commands end up being set to the version that is being released, making cut and paste of those commands less labor intensive and error prone. +IMPORTANT: The following instructions assume that the release manager's < is setup +properly for release and includes a `.glv` file in `gremlin-python` as described in the <> +section, so that the `gremlin-python` module builds in full. + Pre-flight Check ---------------- @@ -46,7 +50,7 @@ and communicating with other members of the community. under release is protected. Tweaks to documentation and other odds and ends related to release are still allowed during this period. . At some point during the week: -.. Run the full integration test suite: `mvn clean install -DskipIntegrationTests=false -DincludeNeo4j -DglvPython` +.. Run the full integration test suite: `mvn clean install -DskipIntegrationTests=false -DincludeNeo4j` .. Deploy a final SNAPSHOT to the snapshot repository. .. Review LICENSE and NOTICE files to make sure that no <>. .. Review javadoc filters on the "Core API" docs to be sure nothing needs to change. @@ -70,7 +74,7 @@ A release candidate is an unofficial release that is represented by a tagged ver offered in cases where there is significant change in a particular version and the potential for upgrades and problems might be high. -. `mvn clean install -DincludeNeo4j -DglvPython` +. `mvn clean install -DincludeNeo4j` .. `mvn verify -DskipIntegrationTests=false -DincludeNeo4j` .. `mvn verify -DskipPerformanceTests=false` . `bin/publish-docs.sh ` - note that under a release candidate the documentation is published as SNAPSHOT @@ -147,14 +151,14 @@ Release & Promote ----------------- . Login to link:https://repository.apache.org/[Apache Nexus] and release the previously closed repository. -. Deploy to link:https://pypi.python.org/pypi[pypi] with `mvn clean install -DskipTests -DglvPython -Dpypi`. It is likely necessary -that this build will occur from the tag for the release, so be sure to checkout the tag first before executing this step. +. Deploy to link:https://pypi.python.org/pypi[pypi] +.. This build will likely occur from the tag for the release, so be sure to checkout the tag first before executing this step. +.. `mvn clean install -DskipTests` +.. `mvn deploy -pl gremlin-python -DskipTests -Dpypi` . `svn co --depth empty https://dist.apache.org/repos/dist/dev/tinkerpop dev; svn up dev/xx.yy.zz` . `svn co --depth empty https://dist.apache.org/repos/dist/release/tinkerpop release; mkdir release/xx.yy.zz` . Copy release files from `dev/xx.yy.zz` to `release/xx.yy.zz`. . `cd release; svn add xx.yy.zz/; svn ci -m "TinkerPop xx.yy.zz release"` -. Deploy `gremlin-python` to pypi with `mvn deploy -pl gremlin-python -Dpypi -DskipTests` (note that `gremlin-python` -should be built with `-DglvPython` first without skipping tests so that the distribution will be present) . Update homepage with references to latest distribution and to other internal links elsewhere on the page. . Wait for Apache Sonatype to sync the artifacts to Maven Central at (link:http://repo1.maven.org/maven2/org/apache/tinkerpop/tinkerpop/[http://repo1.maven.org/maven2/org/apache/tinkerpop/tinkerpop/]). . Wait for zip distributions to to sync to the Apache mirrors (i.e ensure the download links work from a mirror). From 64670636bdfce80a07f8a7543b78f3efdfeb53dd Mon Sep 17 00:00:00 2001 From: Stephen Mallette Date: Wed, 14 Sep 2016 15:58:14 -0400 Subject: [PATCH 7/8] Minor change to AWK to allow the docs to generate properly This provides a solution to TINKERPOP-1431 --- docs/preprocessor/preprocess-file.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/preprocessor/preprocess-file.sh b/docs/preprocessor/preprocess-file.sh index 1069928e57b..1e54c8ee4d9 100755 --- a/docs/preprocessor/preprocess-file.sh +++ b/docs/preprocessor/preprocess-file.sh @@ -133,7 +133,7 @@ if [ ! ${SKIP} ] && [ $(grep -c '^\[gremlin' ${input}) -gt 0 ]; then sed 's/\t/ /g' ${input} | awk -f ${AWK_SCRIPTS}/prepare.awk | - awk -f ${AWK_SCRIPTS}/init-code-blocks.awk -v TP_HOME="${TP_HOME}" PYTHONPATH="." | + awk -f ${AWK_SCRIPTS}/init-code-blocks.awk -v TP_HOME="${TP_HOME}" -v PYTHONPATH="./gremlin-python/target/classes/Lib" | awk -f ${AWK_SCRIPTS}/progressbar.awk -v tpl=${AWK_SCRIPTS}/progressbar.groovy.template | HADOOP_GREMLIN_LIBS="${CONSOLE_HOME}/ext/giraph-gremlin/lib:${CONSOLE_HOME}/ext/tinkergraph-gremlin/lib" bin/gremlin.sh | ${lb} awk -f ${AWK_SCRIPTS}/ignore.awk | From a569ea823e2cc179352f1c76dd2959ac9529ee1c Mon Sep 17 00:00:00 2001 From: Daniel Kuppitz Date: Wed, 14 Sep 2016 22:24:32 +0200 Subject: [PATCH 8/8] Use an absolute path for `PYTHONPATH`. --- docs/preprocessor/preprocess-file.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/preprocessor/preprocess-file.sh b/docs/preprocessor/preprocess-file.sh index 1e54c8ee4d9..c7b4f5e2790 100755 --- a/docs/preprocessor/preprocess-file.sh +++ b/docs/preprocessor/preprocess-file.sh @@ -133,7 +133,7 @@ if [ ! ${SKIP} ] && [ $(grep -c '^\[gremlin' ${input}) -gt 0 ]; then sed 's/\t/ /g' ${input} | awk -f ${AWK_SCRIPTS}/prepare.awk | - awk -f ${AWK_SCRIPTS}/init-code-blocks.awk -v TP_HOME="${TP_HOME}" -v PYTHONPATH="./gremlin-python/target/classes/Lib" | + awk -f ${AWK_SCRIPTS}/init-code-blocks.awk -v TP_HOME="${TP_HOME}" -v PYTHONPATH="${TP_HOME}/gremlin-python/target/classes/Lib" | awk -f ${AWK_SCRIPTS}/progressbar.awk -v tpl=${AWK_SCRIPTS}/progressbar.groovy.template | HADOOP_GREMLIN_LIBS="${CONSOLE_HOME}/ext/giraph-gremlin/lib:${CONSOLE_HOME}/ext/tinkergraph-gremlin/lib" bin/gremlin.sh | ${lb} awk -f ${AWK_SCRIPTS}/ignore.awk |