STORM-1202: Migrate APIs to org.apache.storm, but try to provide some…

… form of backwards compatability

bin/storm.py

@@ -22,6 +22,8 @@
 import subprocess as sub
 import re
 import shlex
+import tempfile
+import uuid
 try:
     # python 3
     from urllib.parse import quote_plus
@@ -221,13 +223,27 @@ def jar(jarfile, klass, *args):
     (http://storm.apache.org/apidocs/backtype/storm/StormSubmitter.html)
     will upload the jar at topology-jar-path when the topology is submitted.
     """
-    exec_storm_class(
-        klass,
-        jvmtype="-client",
-        extrajars=[jarfile, USER_CONF_DIR, STORM_BIN_DIR],
-        args=args,
-        daemon=False,
-        jvmopts=JAR_JVM_OPTS + ["-Dstorm.jar=" + jarfile])
+    transform_class = confvalue("client.jartransformer.class", [CLUSTER_CONF_DIR])
+    if (transform_class != None and transform_class != "nil"):
+        tmpjar = os.path.join(tempfile.gettempdir(), uuid.uuid1().hex+".jar")
+        exec_storm_class("backtype.storm.daemon.ClientJarTransformerRunner", args=[transform_class, jarfile, tmpjar], fork=True, daemon=False)
+        exec_storm_class(
+            klass,
+            jvmtype="-client",
+            extrajars=[tmpjar, USER_CONF_DIR, STORM_BIN_DIR],
+            args=args,
+            daemon=False,
+            fork=True,
+            jvmopts=JAR_JVM_OPTS + ["-Dstorm.jar=" + tmpjar])
+        os.remove(tmpjar)
+    else:
+        exec_storm_class(
+            klass,
+            jvmtype="-client",
+            extrajars=[jarfile, USER_CONF_DIR, STORM_BIN_DIR],
+            args=args,
+            daemon=False,
+            jvmopts=JAR_JVM_OPTS + ["-Dstorm.jar=" + jarfile])
 
 def sql(sql_file, topo_nam):
     """Syntax: [storm sql sql-file topology]

dev-tools/cleanup.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+BASE="$1"
+git reset HEAD -- "$BASE"
+git checkout -- "$BASE"
+git clean -fdx

dev-tools/move_package.sh

@@ -0,0 +1,69 @@
+#!/bin/sh -x
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -u
+set -e
+
+BASE="$1"
+find "$BASE" -type f -print0 | xargs -0 egrep -l 'backtype.storm|storm.trident|storm.starter|storm.kafka|"backtype", "storm"' | egrep -v '.git/|docs/|CHANGELOG.md|dev-tools/move_package.sh|StormShadeRequest.java' | xargs sed -i '' -e 's/storm\(.\)trident/org\1apache\1storm\1trident/g' -e 's/backtype\(.\)storm/org\1apache\1storm/g' -e 's/storm\([\.\\]\)starter/org\1apache\1storm\1starter/g' -e 's/storm\([\.\\]\)kafka/org\1apache\1storm\1kafka/g' -e 's/"backtype", "storm"/"org", "apache", "storm"/g'
+#find "$BASE" -type f -print0 | xargs -0 egrep -l 'backtype.storm|storm.trident' | egrep -v '.git/|docs/|CHANGELOG.md' | xargs sed -i '' -e 's/storm\(.\)trident/org\1apache\1storm\1trident/g' -e 's/backtype\(.\)storm/org\1apache\1storm/g'
+mkdir -p "$BASE"/storm-core/src/jvm/org/apache/storm/ "$BASE"/storm-core/src/clj/org/apache/storm/ "$BASE"/storm-core/test/jvm/org/apache/storm/ "$BASE"/storm-core/test/clj/org/apache/storm/
+#STORM-CORE
+#SRC JVM
+git mv "$BASE"/storm-core/src/jvm/backtype/storm/* "$BASE"/storm-core/src/jvm/org/apache/storm/
+rm -rf "$BASE"/storm-core/src/jvm/backtype
+git mv "$BASE"/storm-core/src/jvm/storm/trident "$BASE"/storm-core/src/jvm/org/apache/storm
+rm -rf "$BASE"/storm-core/src/jvm/storm
+
+#SRC CLJ
+git mv "$BASE"/storm-core/src/clj/backtype/storm/* "$BASE"/storm-core/src/clj/org/apache/storm/
+rm -rf "$BASE"/storm-core/src/clj/backtype
+git mv "$BASE"/storm-core/src/clj/storm/trident "$BASE"/storm-core/src/clj/org/apache/storm
+rm -rf "$BASE"/storm-core/src/clj/storm
+
+#TEST JVM
+git mv "$BASE"/storm-core/test/jvm/backtype/storm/* "$BASE"/storm-core/test/jvm/org/apache/storm/
+rm -rf "$BASE"/storm-core/test/jvm/backtype
+#git mv "$BASE"/storm-core/test/jvm/storm/trident "$BASE"/storm-core/test/jvm/org/apache/storm
+#rm -rf "$BASE"/storm-core/test/jvm/storm
+
+#TEST CLJ
+git mv "$BASE"/storm-core/test/clj/backtype/storm/* "$BASE"/storm-core/test/clj/org/apache/storm/
+rm -rf "$BASE"/storm-core/test/clj/backtype
+git mv "$BASE"/storm-core/test/clj/storm/trident "$BASE"/storm-core/test/clj/org/apache/storm
+rm -rf "$BASE"/storm-core/test/clj/storm
+
+#STORM-STARTER
+mkdir -p "$BASE"/examples/storm-starter/src/jvm/org/apache/ "$BASE"/examples/storm-starter/src/clj/org/apache/ "$BASE"/examples/storm-starter/test/jvm/org/apache/
+#SRC JVM
+git mv "$BASE"/examples/storm-starter/src/jvm/storm "$BASE"/examples/storm-starter/src/jvm/org/apache/
+
+#SRC CLJ
+git mv "$BASE"/examples/storm-starter/src/clj/storm "$BASE"/examples/storm-starter/src/clj/org/apache/
+
+#TEST JVM
+git mv "$BASE"/examples/storm-starter/test/jvm/storm "$BASE"/examples/storm-starter/test/jvm/org/apache/
+
+
+#STORM-KAFKA
+mkdir -p "$BASE"/external/storm-kafka/src/jvm/org/apache/ "$BASE"/external/storm-kafka/src/test/org/apache/
+
+#SRC JVM
+git mv "$BASE"/external/storm-kafka/src/jvm/storm "$BASE"/external/storm-kafka/src/jvm/org/apache/
+
+#TEST JVM
+git mv "$BASE"/external/storm-kafka/src/test/storm "$BASE"/external/storm-kafka/src/test/org/apache/
+

pom.xml

@@ -238,6 +238,7 @@
         <module>storm-buildtools/maven-shade-clojure-transformer</module>
         <module>storm-buildtools/storm-maven-plugins</module>
         <module>storm-core</module>
+        <module>storm-rename-hack</module>
         <module>external/storm-kafka</module>
         <module>external/storm-hdfs</module>
         <module>external/storm-hbase</module>

storm-core/src/jvm/backtype/storm/Config.java

@@ -1986,6 +1986,13 @@ public class Config extends HashMap<String, Object> {
     @isInteger
     public static final String NIMBUS_CODE_SYNC_FREQ_SECS = "nimbus.code.sync.freq.secs";
 
+    /**
+     * An implementation of @{link backtype.storm.daemon.JarTransformer} that will can be used to transform a jar
+     * file before storm jar runs with it. Use with extreme caution.
+     */
+    @isString
+    public static final Object CLIENT_JAR_TRANSFORMER = "client.jartransformer.class";
+
     public static void setClasspath(Map conf, String cp) {
         conf.put(Config.TOPOLOGY_CLASSPATH, cp);
     }

storm-core/src/jvm/backtype/storm/daemon/ClientJarTransformerRunner.java

@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package backtype.storm.daemon;
+
+import backtype.storm.utils.Utils;
+
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.InputStream;
+
+/**
+ * Main executable to load and run a jar transformer
+ */
+public class ClientJarTransformerRunner {
+    public static void main(String [] args) throws IOException {
+        JarTransformer transformer = Utils.jarTransformer(args[0]);
+        InputStream in = new FileInputStream(args[1]);
+        OutputStream out = new FileOutputStream(args[2]);
+        transformer.transform(in, out);
+        in.close();
+        out.close();
+    }
+}

storm-core/src/jvm/backtype/storm/daemon/JarTransformer.java

@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package backtype.storm.daemon;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * A plugin that can be used to transform a jar file in nimbus before it
+ * is used by a topology.
+ */
+public interface JarTransformer {
+    public void transform(InputStream input, OutputStream output) throws IOException;
+}

storm-core/src/jvm/backtype/storm/utils/Utils.java

@@ -23,6 +23,7 @@
 import backtype.storm.blobstore.ClientBlobStore;
 import backtype.storm.blobstore.InputStreamWithMeta;
 import backtype.storm.blobstore.LocalFsBlobStore;
+import backtype.storm.daemon.JarTransformer;
 import backtype.storm.generated.AccessControl;
 import backtype.storm.generated.AccessControlType;
 import backtype.storm.generated.AuthorizationException;
@@ -136,6 +137,14 @@ public static Object newInstance(String klass) {
         }
     }
 
+    public static JarTransformer jarTransformer(String klass) {
+        JarTransformer ret = null;
+        if (klass != null) {
+            ret = (JarTransformer)newInstance(klass);
+        }
+        return ret;
+    }
+
     public static byte[] serialize(Object obj) {
         return serializationDelegate.serialize(obj);
     }

storm-dist/binary/pom.xml

@@ -46,6 +46,11 @@
             <artifactId>storm-sql-runtime</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.storm</groupId>
+            <artifactId>storm-rename-hack</artifactId>
+            <version>${project.version}</version>
+        </dependency>
     </dependencies>
 
     <build>

storm-rename-hack/pom.xml

@@ -0,0 +1,107 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+      <artifactId>storm</artifactId>
+      <groupId>org.apache.storm</groupId>
+      <version>0.11.0-SNAPSHOT</version>
+      <relativePath>../pom.xml</relativePath>
+  </parent>
+
+  <groupId>org.apache.storm</groupId>
+  <artifactId>storm-rename-hack</artifactId>
+  <packaging>jar</packaging>
+
+  <name>storm-rename-hack</name>
+
+  <properties>
+    <mavenVersion>3.0</mavenVersion>
+    <asmVersion>5.0.2</asmVersion>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.apache.storm</groupId>
+      <artifactId>storm-core</artifactId>
+      <version>${project.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.ow2.asm</groupId>
+      <artifactId>asm</artifactId>
+      <version>${asmVersion}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.ow2.asm</groupId>
+      <artifactId>asm-commons</artifactId>
+      <version>${asmVersion}</version>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+        <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-shade-plugin</artifactId>
+            <configuration>
+                <keepDependenciesWithProvidedScope>false</keepDependenciesWithProvidedScope>
+                <createDependencyReducedPom>true</createDependencyReducedPom>
+                <artifactSet>
+                    <includes>
+                        <include>com.google.guava:guava</include>
+                        <include>org.ow2.asm:asm</include>
+                        <include>org.ow2.asm:asm-commons</include>
+                    </includes>
+                </artifactSet>
+                <relocations>
+                    <relocation>
+                        <pattern>com.google</pattern>
+                        <shadedPattern>org.apache.storm.hack.shade.com.google</shadedPattern>
+                    </relocation>
+                    <relocation>
+                        <pattern>org.objectweb.asm</pattern>
+                        <shadedPattern>org.apache.storm.hack.shade.org.objectweb.asm</shadedPattern>
+                    </relocation>
+                </relocations>
+            </configuration>
+            <executions>
+                <execution>
+                    <phase>package</phase>
+                    <goals>
+                        <goal>shade</goal>
+                    </goals>
+                    <configuration>
+                        <transformers>
+                            <transformer
+                                    implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+                            <transformer
+                                    implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                            </transformer>
+                        </transformers>
+                    </configuration>
+                </execution>
+            </executions>
+        </plugin>
+    </plugins>
+  </build>
+</project>