From f5ce1f865aaf8fbebd364799dcc9685a96dd7f16 Mon Sep 17 00:00:00 2001 From: zhuol Date: Wed, 29 Jul 2015 10:54:56 -0500 Subject: [PATCH 1/3] STORM-964, add config for worker logwriter to restrict its mem usage --- conf/defaults.yaml | 1 + storm-core/src/clj/backtype/storm/daemon/supervisor.clj | 2 ++ storm-core/src/jvm/backtype/storm/Config.java | 6 ++++++ 3 files changed, 9 insertions(+) diff --git a/conf/defaults.yaml b/conf/defaults.yaml index c3fa3722911..ddc1c26641c 100644 --- a/conf/defaults.yaml +++ b/conf/defaults.yaml @@ -181,6 +181,7 @@ topology.stats.sample.rate: 0.05 topology.builtin.metrics.bucket.size.secs: 60 topology.fall.back.on.java.serialization: true topology.worker.childopts: null +topology.worker.lw.childopts: "-Xmx64m" topology.executor.receive.buffer.size: 1024 #batched topology.executor.send.buffer.size: 1024 #individual messages topology.transfer.buffer.size: 1024 # batched diff --git a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj index be8f682e5e2..cae56f61308 100644 --- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj +++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj @@ -675,6 +675,7 @@ (add-to-classpath topo-classpath)) top-gc-opts (storm-conf TOPOLOGY-WORKER-GC-CHILDOPTS) gc-opts (substitute-childopts (if top-gc-opts top-gc-opts (conf WORKER-GC-CHILDOPTS)) worker-id storm-id port) + topo-worker-lw-childopts (conf TOPOLOGY-WORKER-LW-CHILDOPTS) user (storm-conf TOPOLOGY-SUBMITTER-USER) logging-sensitivity (storm-conf TOPOLOGY-LOGGING-SENSITIVITY "S3") logfilename (logs-filename storm-id port) @@ -687,6 +688,7 @@ {"LD_LIBRARY_PATH" jlp}) command (concat [(java-cmd) "-cp" classpath + topo-worker-lw-childopts (str "-Dlogfile.name=" logfilename) (str "-Dstorm.home=" storm-home) (str "-Dstorm.id=" storm-id) diff --git a/storm-core/src/jvm/backtype/storm/Config.java b/storm-core/src/jvm/backtype/storm/Config.java index 58e2a4a38c8..7472e70a5b9 100644 --- a/storm-core/src/jvm/backtype/storm/Config.java +++ b/storm-core/src/jvm/backtype/storm/Config.java @@ -1220,6 +1220,12 @@ public class Config extends HashMap { public static final String TOPOLOGY_WORKER_GC_CHILDOPTS="topology.worker.gc.childopts"; public static final Object TOPOLOGY_WORKER_GC_CHILDOPTS_SCHEMA = ConfigValidation.StringOrStringListValidator; + /** + * Topology-specific options for the logwriter process of a worker. + */ + public static final String TOPOLOGY_WORKER_LW_CHILDOPTS="topology.worker.lw.childopts"; + public static final Object TOPOLOGY_WORKER_LW_CHILDOPTS_SCHEMA = ConfigValidation.StringOrStringListValidator; + /** * Topology-specific classpath for the worker child process. This is combined to the usual classpath. */ From d9d2bb7321165c26c256a1d3defdd8507191a6b4 Mon Sep 17 00:00:00 2001 From: zhuol Date: Wed, 29 Jul 2015 17:30:24 -0500 Subject: [PATCH 2/3] Change lw to logwriter --- conf/defaults.yaml | 2 +- storm-core/src/clj/backtype/storm/daemon/supervisor.clj | 4 ++-- storm-core/src/jvm/backtype/storm/Config.java | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/conf/defaults.yaml b/conf/defaults.yaml index ddc1c26641c..c44709f6a0e 100644 --- a/conf/defaults.yaml +++ b/conf/defaults.yaml @@ -181,7 +181,7 @@ topology.stats.sample.rate: 0.05 topology.builtin.metrics.bucket.size.secs: 60 topology.fall.back.on.java.serialization: true topology.worker.childopts: null -topology.worker.lw.childopts: "-Xmx64m" +topology.worker.logwriter.childopts: "-Xmx64m" topology.executor.receive.buffer.size: 1024 #batched topology.executor.send.buffer.size: 1024 #individual messages topology.transfer.buffer.size: 1024 # batched diff --git a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj index cae56f61308..eb2a06ade76 100644 --- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj +++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj @@ -675,7 +675,7 @@ (add-to-classpath topo-classpath)) top-gc-opts (storm-conf TOPOLOGY-WORKER-GC-CHILDOPTS) gc-opts (substitute-childopts (if top-gc-opts top-gc-opts (conf WORKER-GC-CHILDOPTS)) worker-id storm-id port) - topo-worker-lw-childopts (conf TOPOLOGY-WORKER-LW-CHILDOPTS) + topo-worker-logwriter-childopts (conf TOPOLOGY-WORKER-LOGWRITER-CHILDOPTS) user (storm-conf TOPOLOGY-SUBMITTER-USER) logging-sensitivity (storm-conf TOPOLOGY-LOGGING-SENSITIVITY "S3") logfilename (logs-filename storm-id port) @@ -688,7 +688,7 @@ {"LD_LIBRARY_PATH" jlp}) command (concat [(java-cmd) "-cp" classpath - topo-worker-lw-childopts + topo-worker-logwriter-childopts (str "-Dlogfile.name=" logfilename) (str "-Dstorm.home=" storm-home) (str "-Dstorm.id=" storm-id) diff --git a/storm-core/src/jvm/backtype/storm/Config.java b/storm-core/src/jvm/backtype/storm/Config.java index 7472e70a5b9..699641cbc11 100644 --- a/storm-core/src/jvm/backtype/storm/Config.java +++ b/storm-core/src/jvm/backtype/storm/Config.java @@ -1223,8 +1223,8 @@ public class Config extends HashMap { /** * Topology-specific options for the logwriter process of a worker. */ - public static final String TOPOLOGY_WORKER_LW_CHILDOPTS="topology.worker.lw.childopts"; - public static final Object TOPOLOGY_WORKER_LW_CHILDOPTS_SCHEMA = ConfigValidation.StringOrStringListValidator; + public static final String TOPOLOGY_WORKER_LOGWRITER_CHILDOPTS="topology.worker.logwriter.childopts"; + public static final Object TOPOLOGY_WORKER_LOGWRITER_CHILDOPTS_SCHEMA = ConfigValidation.StringOrStringListValidator; /** * Topology-specific classpath for the worker child process. This is combined to the usual classpath. From eb6d7383861220eaeb685206d3946899c20506ac Mon Sep 17 00:00:00 2001 From: zhuol Date: Wed, 29 Jul 2015 22:18:39 -0500 Subject: [PATCH 3/3] use storm-conf --- storm-core/src/clj/backtype/storm/daemon/supervisor.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj index eb2a06ade76..ce638ce88c3 100644 --- a/storm-core/src/clj/backtype/storm/daemon/supervisor.clj +++ b/storm-core/src/clj/backtype/storm/daemon/supervisor.clj @@ -675,7 +675,7 @@ (add-to-classpath topo-classpath)) top-gc-opts (storm-conf TOPOLOGY-WORKER-GC-CHILDOPTS) gc-opts (substitute-childopts (if top-gc-opts top-gc-opts (conf WORKER-GC-CHILDOPTS)) worker-id storm-id port) - topo-worker-logwriter-childopts (conf TOPOLOGY-WORKER-LOGWRITER-CHILDOPTS) + topo-worker-logwriter-childopts (storm-conf TOPOLOGY-WORKER-LOGWRITER-CHILDOPTS) user (storm-conf TOPOLOGY-SUBMITTER-USER) logging-sensitivity (storm-conf TOPOLOGY-LOGGING-SENSITIVITY "S3") logfilename (logs-filename storm-id port)