From ea946d04dcb6df8e65dbf16500a361eaaba13432 Mon Sep 17 00:00:00 2001 From: Kishor Patil Date: Wed, 18 Jun 2014 23:58:33 +0000 Subject: [PATCH 1/3] Show node details on errors for STORM-360 on security --- storm-core/src/clj/backtype/storm/cluster.clj | 10 +- .../clj/backtype/storm/daemon/executor.clj | 3 +- .../src/clj/backtype/storm/daemon/nimbus.clj | 5 +- storm-core/src/clj/backtype/storm/ui/core.clj | 62 +++++- .../generated/AuthorizationException.java | 17 ++ .../backtype/storm/generated/Credentials.java | 17 ++ .../backtype/storm/generated/ErrorInfo.java | 188 +++++++++++++++++- storm-core/src/py/__init__.py | 16 ++ storm-core/src/py/storm/DistributedRPC-remote | 18 ++ storm-core/src/py/storm/DistributedRPC.py | 16 ++ .../py/storm/DistributedRPCInvocations-remote | 18 ++ .../src/py/storm/DistributedRPCInvocations.py | 16 ++ storm-core/src/py/storm/Nimbus-remote | 18 ++ storm-core/src/py/storm/Nimbus.py | 16 ++ storm-core/src/py/storm/__init__.py | 16 ++ storm-core/src/py/storm/constants.py | 16 ++ storm-core/src/py/storm/ttypes.py | 44 +++- storm-core/src/storm.thrift | 2 + storm-core/src/ui/public/component.html | 8 +- .../templates/component-page-template.html | 10 +- .../templates/topology-page-template.html | 8 +- .../test/clj/backtype/storm/cluster_test.clj | 8 +- 22 files changed, 498 insertions(+), 34 deletions(-) diff --git a/storm-core/src/clj/backtype/storm/cluster.clj b/storm-core/src/clj/backtype/storm/cluster.clj index baa4f69a46c..2c6f6f3f1ed 100644 --- a/storm-core/src/clj/backtype/storm/cluster.clj +++ b/storm-core/src/clj/backtype/storm/cluster.clj @@ -148,7 +148,7 @@ (remove-storm-base! [this storm-id]) (set-assignment! [this storm-id info]) (remove-storm! [this storm-id]) - (report-error [this storm-id task-id error]) + (report-error [this storm-id task-id node port error]) (errors [this storm-id task-id]) (set-credentials! [this storm-id creds topo-conf]) (credentials [this storm-id callback]) @@ -220,7 +220,7 @@ (when ser (Utils/deserialize ser))) -(defstruct TaskError :error :time-secs) +(defstruct TaskError :error :time-secs :host :port) (defn- parse-error-path [^String p] @@ -399,9 +399,9 @@ (maybe-deserialize (get-data cluster-state (credentials-path storm-id) (not-nil? callback)))) (report-error - [this storm-id component-id error] + [this storm-id component-id node port error] (let [path (error-path storm-id component-id) - data {:time-secs (current-time-secs) :error (stringify-error error)} + data {:time-secs (current-time-secs) :error (stringify-error error) :host node :port port} _ (mkdirs cluster-state path acls) _ (create-sequential cluster-state (str path "/e") (Utils/serialize data) acls) to-kill (->> (get-children cluster-state path false) @@ -419,7 +419,7 @@ (let [data (-> (get-data cluster-state (str path "/" c) false) maybe-deserialize)] (when data - (struct TaskError (:error data) (:time-secs data)) + (struct TaskError (:error data) (:time-secs data) (:host data) (:port data)) ))) ()) ] diff --git a/storm-core/src/clj/backtype/storm/daemon/executor.clj b/storm-core/src/clj/backtype/storm/daemon/executor.clj index bc491d9828f..0279f4c09ff 100644 --- a/storm-core/src/clj/backtype/storm/daemon/executor.clj +++ b/storm-core/src/clj/backtype/storm/daemon/executor.clj @@ -178,7 +178,8 @@ (swap! interval-errors inc) (when (<= @interval-errors max-per-interval) - (cluster/report-error (:storm-cluster-state executor) (:storm-id executor) (:component-id executor) error) + (cluster/report-error (:storm-cluster-state executor) (:storm-id executor) (:component-id executor) + (local-hostname) (.getThisWorkerPort (:worker-context executor)) error) )))) ;; in its own function so that it can be mocked out by tracked topologies diff --git a/storm-core/src/clj/backtype/storm/daemon/nimbus.clj b/storm-core/src/clj/backtype/storm/daemon/nimbus.clj index 9502745c88e..a514e99048e 100644 --- a/storm-core/src/clj/backtype/storm/daemon/nimbus.clj +++ b/storm-core/src/clj/backtype/storm/daemon/nimbus.clj @@ -902,7 +902,10 @@ (defn- get-errors [storm-cluster-state storm-id component-id] (->> (.errors storm-cluster-state storm-id component-id) - (map #(ErrorInfo. (:error %) (:time-secs %))))) + (map #(let [error-info (ErrorInfo. (:error %) (:time-secs %)) + _ (.set_host error-info (:host %)) + _ (.set_port error-info (:port %))] + error-info)))) (defn- thriftify-executor-id [[first-task-id last-task-id]] (ExecutorInfo. (int first-task-id) (int last-task-id))) diff --git a/storm-core/src/clj/backtype/storm/ui/core.clj b/storm-core/src/clj/backtype/storm/ui/core.clj index e177ca7127b..59ef086e159 100644 --- a/storm-core/src/clj/backtype/storm/ui/core.clj +++ b/storm-core/src/clj/backtype/storm/ui/core.clj @@ -278,9 +278,8 @@ (sort-by #(.get_error_time_secs ^ErrorInfo %)) reverse first)] - (if error - (error-subset (.get_error ^ErrorInfo error)) - ""))) + error + )) (defn component-task-summs [^TopologyInfo summ topology id] @@ -321,6 +320,32 @@ (map nil-to-zero) (apply max))) + +(defn get-error-span [error] + (if (and error (< (time-delta (.get_error_time_secs ^ErrorInfo error)) + (* 60 30))) + {:class "red"} + {} + )) + +(defn get-error-data [error] + (if error + (error-subset (.get_error ^ErrorInfo error)) + "" + )) + +(defn get-error-port [error error-host top-id] + (if error + (.get_port ^ErrorInfo error) + "" + )) + +(defn get-error-host [error] + (if error + (.get_host ^ErrorInfo error) + "" + )) + (defn spout-streams-stats [summs include-sys?] (let [stats-seq (get-filled-stats summs)] @@ -559,7 +584,11 @@ :let [stats-seq (get-filled-stats summs) stats (aggregate-spout-streams (aggregate-spout-stats - stats-seq include-sys?))]] + stats-seq include-sys?)) + last-error (most-recent-error (get errors id)) + error-host (get-error-host last-error) + error-port (get-error-port last-error error-host top-id) + ]] {"spoutId" id "executors" (count summs) "tasks" (sum-tasks summs) @@ -568,14 +597,22 @@ "completeLatency" (float-str (get-in stats [:complete-latencies window])) "acked" (get-in stats [:acked window]) "failed" (get-in stats [:failed window]) - "lastError" (most-recent-error (get errors id))})) + "errorHost" error-host + "errorPort" error-port + "errorWorkerLogLink" (worker-log-link error-host error-port top-id) + "lastError" (get-error-data last-error) + })) (defn bolt-comp [top-id summ-map errors window include-sys?] (for [[id summs] summ-map :let [stats-seq (get-filled-stats summs) stats (aggregate-bolt-streams (aggregate-bolt-stats - stats-seq include-sys?))]] + stats-seq include-sys?)) + last-error (most-recent-error (get errors id)) + error-host (get-error-host last-error) + error-port (get-error-port last-error error-host top-id) + ]] {"boltId" id "executors" (count summs) "tasks" (sum-tasks summs) @@ -587,7 +624,11 @@ "processLatency" (float-str (get-in stats [:process-latencies window])) "acked" (get-in stats [:acked window]) "failed" (get-in stats [:failed window]) - "lastError" (most-recent-error (get errors id))})) + "errorHost" error-host + "errorPort" error-port + "errorWorkerLogLink" (worker-log-link error-host error-port top-id) + "lastError" (get-error-data last-error) + })) (defn topology-summary [^TopologyInfo summ] (let [executors (.get_executors summ) @@ -687,13 +728,16 @@ "workerLogLink" (worker-log-link (.get_host e) (.get_port e) topology-id)})) (defn component-errors - [errors-list] + [errors-list topology-id] (let [errors (->> errors-list (sort-by #(.get_error_time_secs ^ErrorInfo %)) reverse)] {"componentErrors" (for [^ErrorInfo e errors] {"time" (date-str (.get_error_time_secs e)) + "errorHost" (.get_host e) + "errorPort" (.get_port e) + "errorWorkerLogLink" (worker-log-link (.get_host e) (.get_port e) topology-id) "error" (.get_error e)})})) (defn spout-stats @@ -802,7 +846,7 @@ summs (component-task-summs summ topology component) spec (cond (= type :spout) (spout-stats window summ component summs include-sys?) (= type :bolt) (bolt-stats window summ component summs include-sys?)) - errors (component-errors (get (.get_errors summ) component))] + errors (component-errors (get (.get_errors summ) component) topology-id)] (assert-authorized-ui-user user *STORM-CONF* topology-conf) (merge {"user" user diff --git a/storm-core/src/jvm/backtype/storm/generated/AuthorizationException.java b/storm-core/src/jvm/backtype/storm/generated/AuthorizationException.java index 9efc9da385e..6f0218e652a 100644 --- a/storm-core/src/jvm/backtype/storm/generated/AuthorizationException.java +++ b/storm-core/src/jvm/backtype/storm/generated/AuthorizationException.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ /** * Autogenerated by Thrift Compiler (0.7.0) * diff --git a/storm-core/src/jvm/backtype/storm/generated/Credentials.java b/storm-core/src/jvm/backtype/storm/generated/Credentials.java index 105cec1d8ba..c4563a76724 100644 --- a/storm-core/src/jvm/backtype/storm/generated/Credentials.java +++ b/storm-core/src/jvm/backtype/storm/generated/Credentials.java @@ -1,3 +1,20 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ /** * Autogenerated by Thrift Compiler (0.7.0) * diff --git a/storm-core/src/jvm/backtype/storm/generated/ErrorInfo.java b/storm-core/src/jvm/backtype/storm/generated/ErrorInfo.java index fdd8effc92b..13b3fad5cc3 100644 --- a/storm-core/src/jvm/backtype/storm/generated/ErrorInfo.java +++ b/storm-core/src/jvm/backtype/storm/generated/ErrorInfo.java @@ -43,14 +43,20 @@ public class ErrorInfo implements org.apache.thrift.TBase byName = new HashMap(); @@ -69,6 +75,10 @@ public static _Fields findByThriftId(int fieldId) { return ERROR; case 2: // ERROR_TIME_SECS return ERROR_TIME_SECS; + case 3: // HOST + return HOST; + case 4: // PORT + return PORT; default: return null; } @@ -110,7 +120,8 @@ public String getFieldName() { // isset id assignments private static final int __ERROR_TIME_SECS_ISSET_ID = 0; - private BitSet __isset_bit_vector = new BitSet(1); + private static final int __PORT_ISSET_ID = 1; + private BitSet __isset_bit_vector = new BitSet(2); public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; static { @@ -119,6 +130,10 @@ public String getFieldName() { new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); tmpMap.put(_Fields.ERROR_TIME_SECS, new org.apache.thrift.meta_data.FieldMetaData("error_time_secs", org.apache.thrift.TFieldRequirementType.REQUIRED, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))); + tmpMap.put(_Fields.HOST, new org.apache.thrift.meta_data.FieldMetaData("host", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))); + tmpMap.put(_Fields.PORT, new org.apache.thrift.meta_data.FieldMetaData("port", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32))); metaDataMap = Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(ErrorInfo.class, metaDataMap); } @@ -146,6 +161,10 @@ public ErrorInfo(ErrorInfo other) { this.error = other.error; } this.error_time_secs = other.error_time_secs; + if (other.is_set_host()) { + this.host = other.host; + } + this.port = other.port; } public ErrorInfo deepCopy() { @@ -157,6 +176,9 @@ public void clear() { this.error = null; set_error_time_secs_isSet(false); this.error_time_secs = 0; + this.host = null; + set_port_isSet(false); + this.port = 0; } public String get_error() { @@ -204,6 +226,51 @@ public void set_error_time_secs_isSet(boolean value) { __isset_bit_vector.set(__ERROR_TIME_SECS_ISSET_ID, value); } + public String get_host() { + return this.host; + } + + public void set_host(String host) { + this.host = host; + } + + public void unset_host() { + this.host = null; + } + + /** Returns true if field host is set (has been assigned a value) and false otherwise */ + public boolean is_set_host() { + return this.host != null; + } + + public void set_host_isSet(boolean value) { + if (!value) { + this.host = null; + } + } + + public int get_port() { + return this.port; + } + + public void set_port(int port) { + this.port = port; + set_port_isSet(true); + } + + public void unset_port() { + __isset_bit_vector.clear(__PORT_ISSET_ID); + } + + /** Returns true if field port is set (has been assigned a value) and false otherwise */ + public boolean is_set_port() { + return __isset_bit_vector.get(__PORT_ISSET_ID); + } + + public void set_port_isSet(boolean value) { + __isset_bit_vector.set(__PORT_ISSET_ID, value); + } + public void setFieldValue(_Fields field, Object value) { switch (field) { case ERROR: @@ -222,6 +289,22 @@ public void setFieldValue(_Fields field, Object value) { } break; + case HOST: + if (value == null) { + unset_host(); + } else { + set_host((String)value); + } + break; + + case PORT: + if (value == null) { + unset_port(); + } else { + set_port((Integer)value); + } + break; + } } @@ -233,6 +316,12 @@ public Object getFieldValue(_Fields field) { case ERROR_TIME_SECS: return Integer.valueOf(get_error_time_secs()); + case HOST: + return get_host(); + + case PORT: + return Integer.valueOf(get_port()); + } throw new IllegalStateException(); } @@ -248,6 +337,10 @@ public boolean isSet(_Fields field) { return is_set_error(); case ERROR_TIME_SECS: return is_set_error_time_secs(); + case HOST: + return is_set_host(); + case PORT: + return is_set_port(); } throw new IllegalStateException(); } @@ -283,6 +376,24 @@ public boolean equals(ErrorInfo that) { return false; } + boolean this_present_host = true && this.is_set_host(); + boolean that_present_host = true && that.is_set_host(); + if (this_present_host || that_present_host) { + if (!(this_present_host && that_present_host)) + return false; + if (!this.host.equals(that.host)) + return false; + } + + boolean this_present_port = true && this.is_set_port(); + boolean that_present_port = true && that.is_set_port(); + if (this_present_port || that_present_port) { + if (!(this_present_port && that_present_port)) + return false; + if (this.port != that.port) + return false; + } + return true; } @@ -300,6 +411,16 @@ public int hashCode() { if (present_error_time_secs) builder.append(error_time_secs); + boolean present_host = true && (is_set_host()); + builder.append(present_host); + if (present_host) + builder.append(host); + + boolean present_port = true && (is_set_port()); + builder.append(present_port); + if (present_port) + builder.append(port); + return builder.toHashCode(); } @@ -331,6 +452,26 @@ public int compareTo(ErrorInfo other) { return lastComparison; } } + lastComparison = Boolean.valueOf(is_set_host()).compareTo(typedOther.is_set_host()); + if (lastComparison != 0) { + return lastComparison; + } + if (is_set_host()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.host, typedOther.host); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(is_set_port()).compareTo(typedOther.is_set_port()); + if (lastComparison != 0) { + return lastComparison; + } + if (is_set_port()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.port, typedOther.port); + if (lastComparison != 0) { + return lastComparison; + } + } return 0; } @@ -363,6 +504,21 @@ public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.t org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); } break; + case 3: // HOST + if (field.type == org.apache.thrift.protocol.TType.STRING) { + this.host = iprot.readString(); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); + } + break; + case 4: // PORT + if (field.type == org.apache.thrift.protocol.TType.I32) { + this.port = iprot.readI32(); + set_port_isSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); + } + break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type); } @@ -384,6 +540,18 @@ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache. oprot.writeFieldBegin(ERROR_TIME_SECS_FIELD_DESC); oprot.writeI32(this.error_time_secs); oprot.writeFieldEnd(); + if (this.host != null) { + if (is_set_host()) { + oprot.writeFieldBegin(HOST_FIELD_DESC); + oprot.writeString(this.host); + oprot.writeFieldEnd(); + } + } + if (is_set_port()) { + oprot.writeFieldBegin(PORT_FIELD_DESC); + oprot.writeI32(this.port); + oprot.writeFieldEnd(); + } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -404,6 +572,22 @@ public String toString() { sb.append("error_time_secs:"); sb.append(this.error_time_secs); first = false; + if (is_set_host()) { + if (!first) sb.append(", "); + sb.append("host:"); + if (this.host == null) { + sb.append("null"); + } else { + sb.append(this.host); + } + first = false; + } + if (is_set_port()) { + if (!first) sb.append(", "); + sb.append("port:"); + sb.append(this.port); + first = false; + } sb.append(")"); return sb.toString(); } diff --git a/storm-core/src/py/__init__.py b/storm-core/src/py/__init__.py index e69de29bb2d..59dd0608afc 100644 --- a/storm-core/src/py/__init__.py +++ b/storm-core/src/py/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/storm-core/src/py/storm/DistributedRPC-remote b/storm-core/src/py/storm/DistributedRPC-remote index 9b7ebd84ecb..41f8e722c4b 100755 --- a/storm-core/src/py/storm/DistributedRPC-remote +++ b/storm-core/src/py/storm/DistributedRPC-remote @@ -1,3 +1,21 @@ +#!/usr/bin/env python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + #!/usr/bin/env python # # Autogenerated by Thrift Compiler (0.7.0) diff --git a/storm-core/src/py/storm/DistributedRPC.py b/storm-core/src/py/storm/DistributedRPC.py index 851ad658672..21a97969e11 100644 --- a/storm-core/src/py/storm/DistributedRPC.py +++ b/storm-core/src/py/storm/DistributedRPC.py @@ -1,3 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # # Autogenerated by Thrift Compiler (0.7.0) # diff --git a/storm-core/src/py/storm/DistributedRPCInvocations-remote b/storm-core/src/py/storm/DistributedRPCInvocations-remote index 5235dfee92d..37f0cb5d1eb 100755 --- a/storm-core/src/py/storm/DistributedRPCInvocations-remote +++ b/storm-core/src/py/storm/DistributedRPCInvocations-remote @@ -1,3 +1,21 @@ +#!/usr/bin/env python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + #!/usr/bin/env python # # Autogenerated by Thrift Compiler (0.7.0) diff --git a/storm-core/src/py/storm/DistributedRPCInvocations.py b/storm-core/src/py/storm/DistributedRPCInvocations.py index 6de22457679..5120f4d9f78 100644 --- a/storm-core/src/py/storm/DistributedRPCInvocations.py +++ b/storm-core/src/py/storm/DistributedRPCInvocations.py @@ -1,3 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # # Autogenerated by Thrift Compiler (0.7.0) # diff --git a/storm-core/src/py/storm/Nimbus-remote b/storm-core/src/py/storm/Nimbus-remote index c184fab4b5d..efd92204bb7 100755 --- a/storm-core/src/py/storm/Nimbus-remote +++ b/storm-core/src/py/storm/Nimbus-remote @@ -1,3 +1,21 @@ +#!/usr/bin/env python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + #!/usr/bin/env python # # Autogenerated by Thrift Compiler (0.7.0) diff --git a/storm-core/src/py/storm/Nimbus.py b/storm-core/src/py/storm/Nimbus.py index e1ee5103dad..a2ee892bd38 100644 --- a/storm-core/src/py/storm/Nimbus.py +++ b/storm-core/src/py/storm/Nimbus.py @@ -1,3 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # # Autogenerated by Thrift Compiler (0.7.0) # diff --git a/storm-core/src/py/storm/__init__.py b/storm-core/src/py/storm/__init__.py index 10e7d40518b..36923819ebf 100644 --- a/storm-core/src/py/storm/__init__.py +++ b/storm-core/src/py/storm/__init__.py @@ -1 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + __all__ = ['ttypes', 'constants', 'Nimbus', 'DistributedRPC', 'DistributedRPCInvocations'] diff --git a/storm-core/src/py/storm/constants.py b/storm-core/src/py/storm/constants.py index 732b3680dcd..432bbb53af9 100644 --- a/storm-core/src/py/storm/constants.py +++ b/storm-core/src/py/storm/constants.py @@ -1,3 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # # Autogenerated by Thrift Compiler (0.7.0) # diff --git a/storm-core/src/py/storm/ttypes.py b/storm-core/src/py/storm/ttypes.py index 8fa34b0e0ee..1bbaf37c733 100644 --- a/storm-core/src/py/storm/ttypes.py +++ b/storm-core/src/py/storm/ttypes.py @@ -1,3 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # # Autogenerated by Thrift Compiler (0.7.0) # @@ -2458,20 +2474,26 @@ class ErrorInfo: Attributes: - error - error_time_secs + - host + - port """ thrift_spec = ( None, # 0 (1, TType.STRING, 'error', None, None, ), # 1 (2, TType.I32, 'error_time_secs', None, None, ), # 2 + (3, TType.STRING, 'host', None, None, ), # 3 + (4, TType.I32, 'port', None, None, ), # 4 ) def __hash__(self): - return 0 + hash(self.error) + hash(self.error_time_secs) + return 0 + hash(self.error) + hash(self.error_time_secs) + hash(self.host) + hash(self.port) - def __init__(self, error=None, error_time_secs=None,): + def __init__(self, error=None, error_time_secs=None, host=None, port=None,): self.error = error self.error_time_secs = error_time_secs + self.host = host + self.port = port def read(self, iprot): if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: @@ -2492,6 +2514,16 @@ def read(self, iprot): self.error_time_secs = iprot.readI32(); else: iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.host = iprot.readString().decode('utf-8') + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I32: + self.port = iprot.readI32(); + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -2510,6 +2542,14 @@ def write(self, oprot): oprot.writeFieldBegin('error_time_secs', TType.I32, 2) oprot.writeI32(self.error_time_secs) oprot.writeFieldEnd() + if self.host is not None: + oprot.writeFieldBegin('host', TType.STRING, 3) + oprot.writeString(self.host.encode('utf-8')) + oprot.writeFieldEnd() + if self.port is not None: + oprot.writeFieldBegin('port', TType.I32, 4) + oprot.writeI32(self.port) + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() diff --git a/storm-core/src/storm.thrift b/storm-core/src/storm.thrift index 679d29e9c57..f807b743e86 100644 --- a/storm-core/src/storm.thrift +++ b/storm-core/src/storm.thrift @@ -164,6 +164,8 @@ struct ClusterSummary { struct ErrorInfo { 1: required string error; 2: required i32 error_time_secs; + 3: optional string host; + 4: optional i32 port; } struct BoltStats { diff --git a/storm-core/src/ui/public/component.html b/storm-core/src/ui/public/component.html index 0951aed3004..c88dd7016d8 100644 --- a/storm-core/src/ui/public/component.html +++ b/storm-core/src/ui/public/component.html @@ -77,6 +77,10 @@

Storm UI

var executorStats = $("#component-executor-stats"); var componentErrors = $("#component-errors"); $.get("/templates/component-page-template.html", function(template) { + componentErrors.append(Mustache.render($(template).filter("#component-errors-template").html(),response)); + if(response["componentErrors"].length > 0) { + $("#component-errors-table").tablesorter({ sortList: [[0,0]], headers: {1: { sorter: "stormtimestr"}}}); + } componentSummary.append(Mustache.render($(template).filter("#component-summary-template").html(),response)); if(response["componentType"] == "spout") { componentStatsDetail.append(Mustache.render($(template).filter("#spout-stats-detail-template").html(),response)); @@ -99,10 +103,6 @@

Storm UI

$("#bolt-executor-table").tablesorter({ sortList: [[0,0]], headers: {}}); } } - componentErrors.append(Mustache.render($(template).filter("#component-errors-template").html(),response)); - if(response["componentErrors"].length > 0) { - $("#component-errors-table").tablesorter({ sortList: [[0,0]], headers: {1: { sorter: "stormtimestr"}}}); - } }); }); }); diff --git a/storm-core/src/ui/public/templates/component-page-template.html b/storm-core/src/ui/public/templates/component-page-template.html index 6ee95ecadb2..f326067635f 100644 --- a/storm-core/src/ui/public/templates/component-page-template.html +++ b/storm-core/src/ui/public/templates/component-page-template.html @@ -32,7 +32,7 @@

Spout stats

{{#spoutSummary}} -{{windowPretty}} +{{windowPretty}} {{transferred}} {{emitted}} {{completeLatency}} @@ -86,7 +86,7 @@

Bolt stats

{{#boltStats}} -{{windowPretty}} +{{windowPretty}} {{emitted}} {{transferred}} {{executeLatency}} @@ -97,6 +97,7 @@

Bolt stats

{{/boltStats}} + -