From 249b4c8119d8d9c7c43f284f9b1bd99d27029bf5 Mon Sep 17 00:00:00 2001 From: twerges Date: Sat, 29 Jan 2011 12:52:44 -0800 Subject: [PATCH 01/31] replaced incorrect account and zone name Strings in false data generation --- .../com/eucalyptus/reporting/storage/FalseDataGenerator.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/storage/FalseDataGenerator.java b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/storage/FalseDataGenerator.java index c39e1fa6f10..30550ba7bcc 100644 --- a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/storage/FalseDataGenerator.java +++ b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/storage/FalseDataGenerator.java @@ -47,10 +47,10 @@ public static void generateFalseData(String remote) for (int j = 0; j < NUM_USERS; j++) { String userId = String.format("user-%d", j); - String accountId = String.format("instance-%d", + String accountId = String.format("account-%d", (j % NUM_ACCOUNTS)); String clusterId = String.format("cluster-%d", (j % NUM_CLUSTERS)); - String zoneId = String.format("account-%d", (j % NUM_ZONES)); + String zoneId = String.format("zone-%d", (j % NUM_ZONES)); for (int k = 0; k < StorageEvent.EventType.values().length; k++) { long sizeMegs = 1024 + (i * k); From 6fd6d1fc0a470563477b0891cedb73470ec2460b Mon Sep 17 00:00:00 2001 From: twerges Date: Sat, 29 Jan 2011 15:44:08 -0500 Subject: [PATCH 02/31] removed unnecessary import that was breaking build --- .../main/java/com/eucalyptus/bootstrap/SystemBootstrapper.java | 1 - 1 file changed, 1 deletion(-) diff --git a/clc/modules/msgs/src/main/java/com/eucalyptus/bootstrap/SystemBootstrapper.java b/clc/modules/msgs/src/main/java/com/eucalyptus/bootstrap/SystemBootstrapper.java index 6c9a4f23c45..64e0431201b 100644 --- a/clc/modules/msgs/src/main/java/com/eucalyptus/bootstrap/SystemBootstrapper.java +++ b/clc/modules/msgs/src/main/java/com/eucalyptus/bootstrap/SystemBootstrapper.java @@ -71,7 +71,6 @@ import com.eucalyptus.records.EventClass; import com.eucalyptus.records.EventRecord; import com.eucalyptus.records.EventType; -import com.eucalyptus.reporting.queue.QueueFactory; import com.eucalyptus.scripting.groovy.GroovyUtil; import com.eucalyptus.system.LogLevels; import com.eucalyptus.util.LogUtil; From a93961459ceb565cf61a985bd7f22f51d31de558 Mon Sep 17 00:00:00 2001 From: twerges Date: Fri, 4 Feb 2011 17:51:25 -0800 Subject: [PATCH 03/31] javadoc --- .../java/com/eucalyptus/reporting/GenerateReport.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/GenerateReport.java b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/GenerateReport.java index 8d7f1720b44..f593e7bcca7 100644 --- a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/GenerateReport.java +++ b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/GenerateReport.java @@ -13,6 +13,13 @@ import com.eucalyptus.reporting.storage.*; import com.eucalyptus.reporting.units.*; +/** + *

GenerateReport is a command-line tool for rapid testing which uses + * jasper reports internally. It takes command-line args from the + * 

runTest.sh
mechanism and writes a resultant report to a file. + * + * @author tom.werges + */ public class GenerateReport { public static enum Format @@ -46,6 +53,8 @@ public String getFileExtension() } + /* Expects jrxml files to be in /tmp and writes results there. + */ private static File baseDir = new File("/tmp/"); private static File storageReportFile = new File(baseDir, "storage.jrxml"); private static File nestedStorageReportFile = new File(baseDir, "nested_storage.jrxml"); From 966b91b7ddedb9ede4b7fec38a69c380f9460d40 Mon Sep 17 00:00:00 2001 From: twerges Date: Fri, 4 Feb 2011 17:51:39 -0800 Subject: [PATCH 04/31] added --- .../eucalyptus/reporting/storage/package.html | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 clc/modules/reporting/src/main/java/com/eucalyptus/reporting/storage/package.html diff --git a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/storage/package.html b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/storage/package.html new file mode 100644 index 00000000000..90105d2c7ab --- /dev/null +++ b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/storage/package.html @@ -0,0 +1,33 @@ + +

The reporting.storage package contains classes for the storage and retrieval +of historical usage information for Walrus/S3. It could be used to generate +timely visual reports, of S3/Walrus usage over some past time period (like a +month), in order to monitor usage trends or to conduct billing. + +

The reporting.storage package stores its accounting information as a series +of snapshots, each of which contains the total resource usage for +a given (owner,account,cluster,availability_zone) combination over some time +period. + +

The reporting.storage package receives events from S3/Walrus indicating + allocation and deallocation of Walrus/S3 objects. Then the reporting.storage +package uses those events to generate snapshots, every n seconds, of +total resource utilization for each (owner,account,cluster,zone) combination. + +

Events are transmitted to the reporting.storage package over a reliable +message queue. Events are transmitted using the reporting.queue package. + +

Reporting data stored by the reporting.storage package can be accessed +using the 

StorageUsageLog
class. This class will allow you to scan +through the usage logs and gather usage information. This is the main class +for accessing reporting.storage information. + +

In addition, there is a 

StorageDisplayDb
class which is built +atop the 
StorageUsageLog
class. The 
StorageDisplayDb
+class adds logic for displayable Strings, unit conversions, and so on. It +emits displayable bean classes which can be used by jasper or other report +generation engines. + +

The reporting.storage package is used by 

Reports
class in the +
www
module, which generates visual reports based on user +selections. From 1e69126d01bdb1ca47c13995813c690aee25c8cc Mon Sep 17 00:00:00 2001 From: twerges Date: Fri, 4 Feb 2011 17:52:01 -0800 Subject: [PATCH 05/31] javadoc --- .../com/eucalyptus/reporting/storage/StorageUsageSummary.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/storage/StorageUsageSummary.java b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/storage/StorageUsageSummary.java index 3abd013e5d6..a0473634db3 100644 --- a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/storage/StorageUsageSummary.java +++ b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/storage/StorageUsageSummary.java @@ -6,7 +6,7 @@ * usage. * *

StorageUsageSummary is not a bean. StorageUsageSummary accumulates - * data through the 

update
method and then returns results of + * data through the 
update
method and then returns resulting * statistics using the various 
get
methods. * * @author tom.werges From 25d6afe8d312872c2ff823bd71cb6a2a581357c8 Mon Sep 17 00:00:00 2001 From: twerges Date: Fri, 4 Feb 2011 17:53:49 -0800 Subject: [PATCH 06/31] added remote network broker param --- .../reporting/queue/QueueBroker.java | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/queue/QueueBroker.java b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/queue/QueueBroker.java index afe39ba74b9..a0bfdfaccaa 100644 --- a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/queue/QueueBroker.java +++ b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/queue/QueueBroker.java @@ -17,18 +17,22 @@ public class QueueBroker private String brokerName; private String brokerDataDir; private String brokerUrl; + //static:(tcp://remoteServer:63636) + private String remoteBrokerUrl; private List destinations; private BrokerService brokerService; private JmsBrokerThread brokerThread; - private QueueBroker(String brokerName, String brokerUrl, String brokerDataDir) + private QueueBroker(String brokerName, String brokerUrl, String remoteBrokerUrl, + String brokerDataDir) { this.brokerName = brokerName; this.brokerUrl = brokerUrl; this.brokerDataDir = brokerDataDir; this.destinations = new ArrayList(); + this.remoteBrokerUrl = remoteBrokerUrl; } private static QueueBroker instance; @@ -36,7 +40,8 @@ private QueueBroker(String brokerName, String brokerUrl, String brokerDataDir) public static QueueBroker getInstance() { if (instance == null) { - return instance = new QueueBroker(DEFAULT_NAME, DEFAULT_URL, DEFAULT_DIR); + return instance = new QueueBroker(DEFAULT_NAME, DEFAULT_URL, null, + DEFAULT_DIR); } return instance; } @@ -50,14 +55,19 @@ public void addDestination(String destName) public void startup() { + /* TODO: Determine our current location. + * Determine where the main reporting component lives. + * Establish a network connector. + */ try { brokerService = new BrokerService(); brokerService.setBrokerName(brokerName); brokerService.setDataDirectory(brokerDataDir); brokerService.addConnector(brokerUrl); - ActiveMQDestination[] dests = new ActiveMQDestination[destinations.size()]; - destinations.toArray(dests); - brokerService.setDestinations(dests); + if (remoteBrokerUrl != null) { + brokerService.addNetworkConnector(remoteBrokerUrl); + } + brokerService.setUseJmx(true); brokerThread = new JmsBrokerThread(brokerService); brokerThread.start(); Thread.sleep(1000); // give the broker a moment to startup; TODO: @@ -94,7 +104,12 @@ public String getBrokerUrl() public static void main(String[] args) throws Exception { - QueueBroker broker = new QueueBroker(DEFAULT_NAME, DEFAULT_URL, DEFAULT_DIR); + String remoteBrokerUrl = null; + if (args.length > 0) { + remoteBrokerUrl = args[0]; + } + QueueBroker broker = new QueueBroker(DEFAULT_NAME, DEFAULT_URL, + remoteBrokerUrl, DEFAULT_DIR); broker.startup(); } From 6ef7cea744e5a45b8305ce8e288b19df25ad66b6 Mon Sep 17 00:00:00 2001 From: twerges Date: Fri, 4 Feb 2011 17:54:16 -0800 Subject: [PATCH 07/31] commentary --- .../java/com/eucalyptus/reporting/queue/QueueBootstrapper.java | 1 - 1 file changed, 1 deletion(-) diff --git a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/queue/QueueBootstrapper.java b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/queue/QueueBootstrapper.java index 2bb9c294437..e889390c221 100644 --- a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/queue/QueueBootstrapper.java +++ b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/queue/QueueBootstrapper.java @@ -17,7 +17,6 @@ public boolean load() throws Exception /* NOTE: a workaround was moved to SystemBootstrapper.init() because * log4j brain damage prevents ActiveMQ broker from starting. */ - // try { // QueueFactory.getInstance().startup(); // log.info("broker started"); From d8b594bae5f2d98521a52e18748f155349163135 Mon Sep 17 00:00:00 2001 From: twerges Date: Fri, 4 Feb 2011 17:54:42 -0800 Subject: [PATCH 08/31] minor changes to debugging print stmts --- .../reporting/storage/StorageDisplayBean.java | 5 +++++ .../reporting/storage/StorageEventPoller.java | 11 ++++++++++- .../eucalyptus/reporting/storage/StorageUsageLog.java | 3 ++- .../reporting/storage/StorageUsageSnapshot.java | 5 +++-- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/storage/StorageDisplayBean.java b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/storage/StorageDisplayBean.java index 743869ef1e2..122a6c340bd 100644 --- a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/storage/StorageDisplayBean.java +++ b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/storage/StorageDisplayBean.java @@ -69,5 +69,10 @@ public Long getObjectsSizeTime() INTERNAL_UNITS.getSizeTimeSizeUnit(), units.getSizeTimeSizeUnit(), INTERNAL_UNITS.getSizeTimeTimeUnit(), units.getSizeTimeTimeUnit()); } + + public Units getUnits() + { + return units; + } } diff --git a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/storage/StorageEventPoller.java b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/storage/StorageEventPoller.java index e8110727ee8..76383e3c6a2 100644 --- a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/storage/StorageEventPoller.java +++ b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/storage/StorageEventPoller.java @@ -10,6 +10,12 @@ import com.eucalyptus.reporting.event.*; import com.eucalyptus.reporting.queue.QueueReceiver; +/** + *

StorageEventPoller is run periodically and polls the message queue for + * new storage events. It then aggregates those events into snapshots. + * + * @author tom.werges + */ public class StorageEventPoller { private static Logger LOG = Logger.getLogger( StorageEventPoller.class ); @@ -26,6 +32,8 @@ public StorageEventPoller(QueueReceiver receiver) public void writeEvents() { + LOG.info("Queue poll:" + System.currentTimeMillis()); + EntityWrapper entityWrapper = EntityWrapper.get( StorageUsageSnapshot.class ); Session sess = null; @@ -58,6 +66,7 @@ public void writeEvents() event = receiver.receiveEventNoWait()) { StorageEvent storageEvent = (StorageEvent) event; + LOG.info("Receive event:" + storageEvent.toString()); UsageDataKey key = new UsageDataKey(storageEvent.getOwnerId(), storageEvent.getAccountId(), storageEvent.getClusterName(), @@ -109,7 +118,7 @@ public void writeEvents() SnapshotKey snapshotKey = key.newSnapshotKey(timestampMs); StorageUsageSnapshot sus = new StorageUsageSnapshot(snapshotKey, usageDataMap.get(key)); - System.out.println("Storing:" + sus); + LOG.info("Save event:" + sus); sess.save(sus); } changedSnapshots.clear(); diff --git a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/storage/StorageUsageLog.java b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/storage/StorageUsageLog.java index 869d3902d70..449e1688cdf 100644 --- a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/storage/StorageUsageLog.java +++ b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/storage/StorageUsageLog.java @@ -11,7 +11,7 @@ import com.eucalyptus.reporting.instance.InstanceAttributes; /** - *

StorageUsageLog + *

StorageUsageLog is the main API for accessing storage usage information. * * @author tom.werges */ @@ -234,6 +234,7 @@ public Map> scanSummarize( SummaryInfo info = innerMap.get(innerCritVal); StorageUsageData lastData = info.getLastData(); long durationSecs = (timestampMs - info.getLastTimestamp()) / 1000; + System.out.println("info:" + info + " summary:" + info.getSummary() + " lastData:" + lastData); info.getSummary().updateValues(lastData.getVolumesMegs(), lastData.getSnapshotsMegs(), lastData.getObjectsMegs(), durationSecs); diff --git a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/storage/StorageUsageSnapshot.java b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/storage/StorageUsageSnapshot.java index 7464c645474..02d18d7d1ab 100644 --- a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/storage/StorageUsageSnapshot.java +++ b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/storage/StorageUsageSnapshot.java @@ -6,7 +6,7 @@ *

StorageUsageSnapshot is a snapshot of disk data usage at some point in * time. * - * @author twerges + * @author tom.werges */ @Entity @PersistenceContext(name="reporting") @@ -22,7 +22,7 @@ protected StorageUsageSnapshot() { } - + public StorageUsageSnapshot(SnapshotKey key, StorageUsageData usageData) { this.key = key; @@ -43,4 +43,5 @@ public String toString() { return String.format("[key:%s,usageData:%s]", this.key, this.usageData); } + } From 6dc8241575deaf679a23b722ed03d3928d4bf122 Mon Sep 17 00:00:00 2001 From: twerges Date: Fri, 4 Feb 2011 17:55:06 -0800 Subject: [PATCH 09/31] minor --- .../eucalyptus/reporting/instance/FalseDataGenerator.java | 4 ++-- .../eucalyptus/reporting/instance/InstanceDisplayBean.java | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/FalseDataGenerator.java b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/FalseDataGenerator.java index 5e7b4b19ec6..cf7ae5261b5 100644 --- a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/FalseDataGenerator.java +++ b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/FalseDataGenerator.java @@ -211,7 +211,7 @@ protected long getCurrentTimeMillis() return fakeCurrentTimeMillis; } } - + public static void main(String[] args) throws Exception { String methodName = args[0]; @@ -226,5 +226,5 @@ public static void main(String[] args) throws Exception FalseDataGenerator.class.getDeclaredMethod(methodName, paramTypes) .invoke(null, methodArgsArray); } - + } diff --git a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceDisplayBean.java b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceDisplayBean.java index 28e0802f987..426faef164d 100644 --- a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceDisplayBean.java +++ b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceDisplayBean.java @@ -97,5 +97,10 @@ public Long getDiskIoSize() return UnitUtil.convertSize(summary.getDiskIoMegs(), INTERNAL_UNITS.getSizeUnit(), units.getSizeUnit()); } + + public Units getUnits() + { + return units; + } } From 8a176e0c235361167535074b2b6628bc8fee316f Mon Sep 17 00:00:00 2001 From: twerges Date: Fri, 4 Feb 2011 17:55:26 -0800 Subject: [PATCH 10/31] added test instance event listener --- .../reporting/ReportingBootstrapper.java | 62 ++++++++++++++++--- 1 file changed, 55 insertions(+), 7 deletions(-) diff --git a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/ReportingBootstrapper.java b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/ReportingBootstrapper.java index cd803b51084..930505b12ed 100644 --- a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/ReportingBootstrapper.java +++ b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/ReportingBootstrapper.java @@ -3,14 +3,13 @@ import java.util.Timer; import java.util.TimerTask; -import org.apache.log4j.*; +import org.apache.log4j.Logger; import com.eucalyptus.bootstrap.*; -import com.eucalyptus.reporting.instance.*; -import com.eucalyptus.reporting.storage.*; +import com.eucalyptus.reporting.instance.InstanceEventListener; import com.eucalyptus.reporting.queue.*; import com.eucalyptus.reporting.queue.QueueFactory.QueueIdentifier; -import com.eucalyptus.system.Threads; +import com.eucalyptus.reporting.storage.StorageEventPoller; @Provides(Component.reporting) @RunDuring(Bootstrap.Stage.RemoteServicesInit) @@ -19,6 +18,12 @@ public class ReportingBootstrapper { private static Logger log = Logger.getLogger( ReportingBootstrapper.class ); + /** + * Sets up listeners etc in test mode whereby they generate fake times for + * incoming data. + */ + private static final boolean TEST = true; //TODO: CHANGE!!! + private static long POLLER_DELAY_MS = 10000l; private StorageEventPoller storagePoller; @@ -82,7 +87,15 @@ public boolean start() /* Start storage receiver and storage queue poller thread */ QueueReceiver storageReceiver = queueFactory.getReceiver(QueueIdentifier.STORAGE); - final StorageEventPoller poller = new StorageEventPoller(storageReceiver); + final StorageEventPoller poller; + //Start fake pollers and listeners if this is a test + if (TEST) { + poller = new TestStorageEventPoller(storageReceiver); + instanceListener = new TestInstanceEventListener(); + } else { + poller = new StorageEventPoller(storageReceiver); + instanceListener = new InstanceEventListener(); + } timer = new Timer(true); timer.schedule(new TimerTask() { @Override @@ -98,7 +111,6 @@ public void run() */ QueueReceiver instanceReceiver = queueFactory.getReceiver(QueueIdentifier.INSTANCE); - instanceListener = new InstanceEventListener(); instanceReceiver.addEventListener(instanceListener); log.info("ReportingBootstrapper started"); @@ -128,8 +140,8 @@ public boolean stop() { try { log.info("ReportingBootstrapper stopped"); - instanceListener.flush(); timer.cancel(); + instanceListener.flush(); storagePoller.writeEvents(); queueFactory.shutdown(); return true; @@ -138,5 +150,41 @@ public boolean stop() return false; } } + + private static final long START_TIME = 1104566400000l; //Jan 1, 2005 12:00AM + private static final int TIME_USAGE_APART = 100000; //ms + private class TestInstanceEventListener + extends InstanceEventListener + { + private long timeMs = START_TIME; + + @Override + protected long getCurrentTimeMillis() + { + long oldTimeMs = timeMs; + timeMs += TIME_USAGE_APART; + return oldTimeMs; + } + } + + private class TestStorageEventPoller + extends StorageEventPoller + { + private long timeMs = START_TIME; + + public TestStorageEventPoller(QueueReceiver receiver) + { + super(receiver); + } + + @Override + protected long getTimestampMs() + { + long oldTimeMs = timeMs; + timeMs += TIME_USAGE_APART; + return oldTimeMs; + } + } + } From e2107cc56f6adb540ef73549b99bfcf7a7397cb7 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 10 Feb 2011 20:16:44 -0800 Subject: [PATCH 11/31] chris commits code for sharing --- clc/Makefile | 27 +++++++++++-- clc/build.xml | 39 +++++++++++++++++-- .../upgrade/StandalonePersistence.java | 4 +- clc/modules/module-inc.xml | 17 ++++++++ .../bootstrap/ServiceJarDiscovery.java | 17 ++++++++ .../component/ComponentDiscovery.java | 4 ++ .../instance/FalseDataGenerator.java | 2 +- .../reporting/storage/FalseDataGenerator.java | 9 +++-- clc/modules/storage-controller/build.xml | 3 ++ clc/modules/www/build.xml | 3 ++ clc/tools/runTest.sh | 2 +- 11 files changed, 112 insertions(+), 15 deletions(-) diff --git a/clc/Makefile b/clc/Makefile index d46b1f34678..d93418e86cd 100644 --- a/clc/Makefile +++ b/clc/Makefile @@ -9,6 +9,7 @@ web_prop_dir = ${TOP}/clc/modules/www/conf .PHONY: all build update_properties update_theme deps extras install all: deps build +test: deps build-test deps: @$(ANT) deps @@ -48,9 +49,22 @@ build: deps extras # this causes problems for pbuilder, should be handled by configure and build.xml already # @if test ! -e deps/cloud-lib.tar.gz; then \ # echo "Cannot find java deps, please run 'make deps' in the 'clc/' subdirectory."; exit 1; fi - @ANT_OPTS="-Xmx1g -XX:MaxPermSize=256m" $(ANT) build-all - @ANT_OPTS="-Xmx1g -XX:MaxPermSize=256m" $(ANT) bind - @ANT_OPTS="-Xmx1g -XX:MaxPermSize=256m" $(ANT) jar + @ANT_OPTS="-Xmx1g -XX:MaxPermSize=512m" $(ANT) build-all + @ANT_OPTS="-Xmx1g -XX:MaxPermSize=512m" $(ANT) bind + @ANT_OPTS="-Xmx1g -XX:MaxPermSize=512m" $(ANT) jar + @make -C modules/storage-controller/native + @make -C modules/bootstrap + +build-test: deps extras + @# we need JAVA_HOME + @if test -z "${JAVA_HOME}" ; then \ + echo "You need to define JAVA_HOME!"; exit 1; fi +# this causes problems for pbuilder, should be handled by configure and build.xml already +# @if test ! -e deps/cloud-lib.tar.gz; then \ +# echo "Cannot find java deps, please run 'make deps' in the 'clc/' subdirectory."; exit 1; fi + @ANT_OPTS="-Xmx1g -XX:MaxPermSize=512m" $(ANT) build-test-all + @ANT_OPTS="-Xmx64m -XX:MaxPermSize=64m" $(ANT) bind + @ANT_OPTS="-Xmx1g -XX:MaxPermSize=512m" $(ANT) jar @make -C modules/storage-controller/native @make -C modules/bootstrap @@ -64,8 +78,13 @@ distclean: clean make -C modules/bootstrap distclean make -C modules/storage-controller/native distclean +install-test: build-test + @ANT_OPTS="-Xmx1g -XX:MaxPermSize=512m" $(ANT) install + @make -C modules/storage-controller/native install + @make -C modules/bootstrap install + install: build - @ANT_OPTS="-Xmx1g -XX:MaxPermSize=256m" $(ANT) install + @ANT_OPTS="-Xmx1g -XX:MaxPermSize=512m" $(ANT) install @make -C modules/storage-controller/native install @make -C modules/bootstrap install diff --git a/clc/build.xml b/clc/build.xml index 33f0c6bbec3..1ac7d55e566 100644 --- a/clc/build.xml +++ b/clc/build.xml @@ -119,7 +119,40 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -158,9 +191,9 @@ - + diff --git a/clc/modules/core/src/main/java/com/eucalyptus/upgrade/StandalonePersistence.java b/clc/modules/core/src/main/java/com/eucalyptus/upgrade/StandalonePersistence.java index b7619ea2616..f3aab24666a 100644 --- a/clc/modules/core/src/main/java/com/eucalyptus/upgrade/StandalonePersistence.java +++ b/clc/modules/core/src/main/java/com/eucalyptus/upgrade/StandalonePersistence.java @@ -195,8 +195,8 @@ static void setupSystemProperties( ) { newLibDir = getAndCheckLibDirectory( eucaHome ); } - private static void setupConfigurations( ) { - ServiceJarDiscovery.runDiscovery( new ComponentDiscovery( ) ); + static void setupConfigurations( ) { + ServiceJarDiscovery.doSingleDiscovery( new ComponentDiscovery( ) ); } private static File getAndCheckLibDirectory( String eucaHome ) { diff --git a/clc/modules/module-inc.xml b/clc/modules/module-inc.xml index 40aa08a0ed0..603e28e2901 100644 --- a/clc/modules/module-inc.xml +++ b/clc/modules/module-inc.xml @@ -88,6 +88,7 @@ + @@ -159,6 +160,22 @@ + + + + + + + + + + + + + + + + diff --git a/clc/modules/msgs/src/main/java/com/eucalyptus/bootstrap/ServiceJarDiscovery.java b/clc/modules/msgs/src/main/java/com/eucalyptus/bootstrap/ServiceJarDiscovery.java index 3d829180339..cb602a62f29 100644 --- a/clc/modules/msgs/src/main/java/com/eucalyptus/bootstrap/ServiceJarDiscovery.java +++ b/clc/modules/msgs/src/main/java/com/eucalyptus/bootstrap/ServiceJarDiscovery.java @@ -84,6 +84,23 @@ private static void doDiscovery( ) { ServiceJarDiscovery.runDiscovery( ); } + public static void doSingleDiscovery( ServiceJarDiscovery s ) { + File libDir = new File( BaseDirectory.LIB.toString( ) ); + for ( File f : libDir.listFiles( ) ) { + if ( f.getName( ).startsWith( "eucalyptus" ) && f.getName( ).endsWith( ".jar" ) + && !f.getName( ).matches( ".*-ext-.*" ) ) { + LOG.debug( "Found eucalyptus component jar: " + f.getName( ) ); + try { + ServiceJarDiscovery.processFile( f ); + } catch ( Throwable e ) { + LOG.error( e.getMessage( ) ); + continue; + } + } + } + ServiceJarDiscovery.runDiscovery( s ); + } + public static void checkUniqueness( Class c ) { if ( classList.get( c ).size( ) > 1 ) { diff --git a/clc/modules/msgs/src/main/java/com/eucalyptus/component/ComponentDiscovery.java b/clc/modules/msgs/src/main/java/com/eucalyptus/component/ComponentDiscovery.java index 66254308a0c..6ee15bc015a 100644 --- a/clc/modules/msgs/src/main/java/com/eucalyptus/component/ComponentDiscovery.java +++ b/clc/modules/msgs/src/main/java/com/eucalyptus/component/ComponentDiscovery.java @@ -76,11 +76,15 @@ public boolean processClass( Class candidate ) throws Throwable { if ( ComponentId.class.isAssignableFrom( candidate ) && !Modifier.isAbstract( candidate.getModifiers( ) ) && !Modifier.isInterface( candidate.getModifiers( ) ) && !Any.class.equals( candidate ) ) { try { + LOG.info( "Found ComponentId of type: " + candidate.getCanonicalName() ); ComponentId id = ( ComponentId ) candidate.newInstance( ); ComponentIds.register( id ); + LOG.info( "-> Registering ComponentId of type: " + candidate.getCanonicalName() ); Components.create( id ); + } catch ( Throwable ex ) { LOG.error( ex, ex ); + LOG.info( "Error occurred while trying to register ComponentId of type: " + ex.getMessage(), ex ); } return true; } else { diff --git a/clc/modules/reporting/src/test/java/com/eucalyptus/reporting/instance/FalseDataGenerator.java b/clc/modules/reporting/src/test/java/com/eucalyptus/reporting/instance/FalseDataGenerator.java index 5e7b4b19ec6..d7f0c025990 100644 --- a/clc/modules/reporting/src/test/java/com/eucalyptus/reporting/instance/FalseDataGenerator.java +++ b/clc/modules/reporting/src/test/java/com/eucalyptus/reporting/instance/FalseDataGenerator.java @@ -96,7 +96,7 @@ public static void generateFalseData(String remote) new Long(diskIoMegs)); System.out.println("Generating:" + i); if (isLocal) { - listener.receiveEvent(event); + listener.fireEvent(event); } else { queueSender.send(event); } diff --git a/clc/modules/reporting/src/test/java/com/eucalyptus/reporting/storage/FalseDataGenerator.java b/clc/modules/reporting/src/test/java/com/eucalyptus/reporting/storage/FalseDataGenerator.java index 30550ba7bcc..b35ff59a6cf 100644 --- a/clc/modules/reporting/src/test/java/com/eucalyptus/reporting/storage/FalseDataGenerator.java +++ b/clc/modules/reporting/src/test/java/com/eucalyptus/reporting/storage/FalseDataGenerator.java @@ -2,10 +2,11 @@ import java.util.*; +import com.eucalyptus.event.EventListener; import com.eucalyptus.reporting.GroupByCriterion; import com.eucalyptus.reporting.Period; -import com.eucalyptus.reporting.event.*; -import com.eucalyptus.reporting.event.EventListener; +import com.eucalyptus.reporting.event.Event; +import com.eucalyptus.reporting.event.StorageEvent; import com.eucalyptus.reporting.queue.*; import com.eucalyptus.reporting.queue.QueueFactory.QueueIdentifier; @@ -88,13 +89,13 @@ public void send(Event e) } @Override - public void addEventListener(EventListener el) + public void addEventListener(EventListener el) { // empty } @Override - public void removeEventListener(EventListener el) + public void removeEventListener(EventListener el) { // empty } diff --git a/clc/modules/storage-controller/build.xml b/clc/modules/storage-controller/build.xml index 6c1fd79e755..7a603b2fb78 100644 --- a/clc/modules/storage-controller/build.xml +++ b/clc/modules/storage-controller/build.xml @@ -67,6 +67,9 @@ + + + diff --git a/clc/modules/www/build.xml b/clc/modules/www/build.xml index 5fd4d42315d..6524c6d789d 100644 --- a/clc/modules/www/build.xml +++ b/clc/modules/www/build.xml @@ -72,6 +72,9 @@ + + + diff --git a/clc/tools/runTest.sh b/clc/tools/runTest.sh index 8327479cb99..a970970aadf 100755 --- a/clc/tools/runTest.sh +++ b/clc/tools/runTest.sh @@ -30,7 +30,7 @@ for FILE in $FILES; do export CLASSPATH=${FILE}:${CLASSPATH} done CLASSPATH=${EUCALYPTUS}/etc/eucalyptus/cloud.d/upgrade:${EUCALYPTUS}/etc/eucalyptus/cloud.d/scripts:${CLASSPATH} - +echo -e "${CLASSPATH//:/\n}" java -Xbootclasspath/p:${EUCALYPTUS}/usr/share/eucalyptus/openjdk-crypto.jar -classpath ${CLASSPATH} \ -Deuca.home=${EUCALYPTUS} \ -Deuca.lib.dir=${EUCALYPTUS} \ From 39f7739e80f0ca3dad8e89a5ff887e7d6f19d688 Mon Sep 17 00:00:00 2001 From: twerges Date: Mon, 21 Feb 2011 13:20:08 -0800 Subject: [PATCH 12/31] changing incorrect branch name; dont know how this got in here --- clc/.project | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clc/.project b/clc/.project index 5a42a3d575d..7b2d3b393e1 100644 --- a/clc/.project +++ b/clc/.project @@ -1,6 +1,6 @@ - grze_eee + eee From 20ff1896031f1aa332679b56f255e6bf00709228 Mon Sep 17 00:00:00 2001 From: twerges Date: Mon, 21 Feb 2011 13:20:27 -0800 Subject: [PATCH 13/31] formatting and minor modifications --- .../com/eucalyptus/reporting/instance/InstanceUsageLog.java | 4 ++-- .../com/eucalyptus/reporting/storage/StorageDisplayDb.java | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceUsageLog.java b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceUsageLog.java index 4cc36b90ce9..c0dbb69311a 100644 --- a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceUsageLog.java +++ b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceUsageLog.java @@ -1,7 +1,7 @@ package com.eucalyptus.reporting.instance; -import org.hibernate.*; -import org.apache.log4j.*; +import org.apache.log4j.Logger; +import org.hibernate.Session; import com.eucalyptus.configurable.ConfigurableClass; import com.eucalyptus.entities.EntityWrapper; diff --git a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/storage/StorageDisplayDb.java b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/storage/StorageDisplayDb.java index 828403b0045..cbe479daf22 100644 --- a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/storage/StorageDisplayDb.java +++ b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/storage/StorageDisplayDb.java @@ -15,7 +15,7 @@ private StorageDisplayDb(StorageUsageLog usageLog) } private static StorageDisplayDb instance = null; - + public static StorageDisplayDb getInstance() { if (instance == null) { @@ -23,7 +23,7 @@ public static StorageDisplayDb getInstance() } return instance; } - + public List search(Period period, GroupByCriterion criterion, Units displayUnits) { @@ -37,7 +37,7 @@ public List search(Period period, GroupByCriterion criterion } return items; } - + public List searchGroupBy( Period period, GroupByCriterion groupByCrit, GroupByCriterion crit, Units displayUnits) From c9b72925f4b54b4ef6bc4dcc844bfeed15d10331 Mon Sep 17 00:00:00 2001 From: twerges Date: Tue, 8 Mar 2011 17:43:48 -0800 Subject: [PATCH 14/31] re-added hacky broker startup --- .../bootstrap/SystemBootstrapper.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/clc/modules/msgs/src/main/java/com/eucalyptus/bootstrap/SystemBootstrapper.java b/clc/modules/msgs/src/main/java/com/eucalyptus/bootstrap/SystemBootstrapper.java index 51d9a5b20ef..b2012c91fa8 100644 --- a/clc/modules/msgs/src/main/java/com/eucalyptus/bootstrap/SystemBootstrapper.java +++ b/clc/modules/msgs/src/main/java/com/eucalyptus/bootstrap/SystemBootstrapper.java @@ -125,6 +125,18 @@ public boolean init( ) throws Exception { if ( LogLevels.EXTREME ) { System.setProperty( "euca.log.level", "TRACE" ); } + + /* This is a workaround for log4j brain damage which prevented the + * ActiveMQ broker from working properly. + */ + try { + GroovyUtil.eval("com.eucalyptus.reporting.queue.QueueBroker.getInstance().startup()"); + LOG.info("Groovy eval of queue factory startup succeeded."); + } catch (Exception ex) { + LOG.error("Groovy eval of queue factory startup failed.", ex); + } + + System.setOut( new PrintStream( System.out ) { public void print( final String string ) { if ( string.replaceAll( "\\s*", "" ).length( ) > 2 ) { @@ -132,7 +144,12 @@ public void print( final String string ) { } } } + + + ); + + System.setErr( new PrintStream( System.err ) { public void print( final String string ) { if ( string.replaceAll( "\\s*", "" ).length( ) > 2 ) { @@ -141,6 +158,7 @@ public void print( final String string ) { } } ); + LOG.info( LogUtil.subheader( "Starting system with debugging set as: " + Join.join( "\n", LogLevels.class.getDeclaredFields( ) ) ) ); try { Logger.getLogger( "com.eucalyptus.entities.EntityWrapper" ).fatal( "Starting up" ); From 3aa86de7f1b8554248d475ed41fea5d5b0bd9e64 Mon Sep 17 00:00:00 2001 From: twerges Date: Tue, 15 Mar 2011 15:02:23 -0700 Subject: [PATCH 15/31] Added ability to run over broker, start broker, and generate fake times thru injected TestEventListener --- .../instance/FalseDataGenerator.java | 77 +++++++++++++------ 1 file changed, 54 insertions(+), 23 deletions(-) diff --git a/clc/modules/reporting/src/test/java/com/eucalyptus/reporting/instance/FalseDataGenerator.java b/clc/modules/reporting/src/test/java/com/eucalyptus/reporting/instance/FalseDataGenerator.java index d7f0c025990..a5b835bcb7d 100644 --- a/clc/modules/reporting/src/test/java/com/eucalyptus/reporting/instance/FalseDataGenerator.java +++ b/clc/modules/reporting/src/test/java/com/eucalyptus/reporting/instance/FalseDataGenerator.java @@ -2,8 +2,10 @@ import java.util.*; +import com.eucalyptus.reporting.ReportingBootstrapper; import com.eucalyptus.reporting.GroupByCriterion; import com.eucalyptus.reporting.Period; +import com.eucalyptus.reporting.event.Event; import com.eucalyptus.reporting.event.InstanceEvent; import com.eucalyptus.reporting.queue.QueueFactory; import com.eucalyptus.reporting.queue.QueueSender; @@ -11,13 +13,13 @@ import com.eucalyptus.upgrade.TestDescription; /** - *

FalseDataGenerator generates false data about instances. - * It generates fake starting and ending times, imaginary resource - * usage, fictitious clusters, and non-existent accounts and users. + *

FalseDataGenerator generates false data about instances. It generates + * fake starting and ending times, imaginary resource usage, fictitious + * clusters, and non-existent accounts and users. * - *

The main use of this class is to be called from the - * 

clc/tools/runTest.sh
tool, specifying the class name and a - * method below. + *

FalseDataGenerator is meant to be called from the command-line tool, + * 

clc/tools/runTest.sh
, specifying the class name and a method + * below. * *

False data should be deleted afterward by calling the * 

deleteFalseData
method. @@ -39,6 +41,8 @@ public class FalseDataGenerator private static final int NUM_ACCOUNT = 16; private static final int NUM_CLUSTER = 4; private static final int NUM_AVAIL_ZONE = 2; + + private static ReportingBootstrapper reportingBootstrapper = null; private enum FalseInstanceType { @@ -52,11 +56,27 @@ private enum FalseInstanceType public static void generateFalseData(String remote) { boolean isLocal = (remote != null && remote.equalsIgnoreCase("local")); + + /* Slightly confusing here... + * + */ + QueueSender queueSender = null; + TestEventListener listener = new TestEventListener(); + listener.setCurrentTimeMillis(START_TIME); + if (isLocal) { + queueSender = listener; + } else { + reportingBootstrapper = new ReportingBootstrapper(); + reportingBootstrapper.setOverriddenInstanceEventListener(listener); + reportingBootstrapper.start(); + queueSender = QueueFactory.getInstance().getSender(QueueIdentifier.INSTANCE); + } + System.out.println(" ----> GENERATING FALSE DATA " + (isLocal ? "(local)" : "(remote)")); - + List fakeInstances = new ArrayList(); - + for (int i = 0; i < NUM_INSTANCE; i++) { String uuid = new Long(i).toString(); @@ -72,18 +92,11 @@ public static void generateFalseData(String remote) instanceId, instanceType, userId, accountId, clusterName, availZone); fakeInstances.add(insAttrs); - } - QueueSender queueSender = null; - TestEventListener listener = null; - if (isLocal) { - listener = new TestEventListener(); - listener.setCurrentTimeMillis(START_TIME); - } else { - queueSender = QueueFactory.getInstance().getSender(QueueIdentifier.INSTANCE); } + for (int i=0; i Date: Tue, 15 Mar 2011 15:04:45 -0700 Subject: [PATCH 16/31] removed option to use local generation as this was confusing and made no sense, now that we have the queue running always and the queue can be used locally --- .../instance/FalseDataGenerator.java | 27 ++++++------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/clc/modules/reporting/src/test/java/com/eucalyptus/reporting/instance/FalseDataGenerator.java b/clc/modules/reporting/src/test/java/com/eucalyptus/reporting/instance/FalseDataGenerator.java index a5b835bcb7d..1ca6a8b2ce1 100644 --- a/clc/modules/reporting/src/test/java/com/eucalyptus/reporting/instance/FalseDataGenerator.java +++ b/clc/modules/reporting/src/test/java/com/eucalyptus/reporting/instance/FalseDataGenerator.java @@ -49,30 +49,19 @@ private enum FalseInstanceType M1SMALL, C1MEDIUM, M1LARGE, M1XLARGE, C1XLARGE; } - /** - * @param remote Must equal "local" if this is to be run locally; any other - * value means it will run over the MQ and will search for a broker. - */ - public static void generateFalseData(String remote) + public static void generateFalseData() { - boolean isLocal = (remote != null && remote.equalsIgnoreCase("local")); + System.out.println(" ----> GENERATING FALSE DATA"); - /* Slightly confusing here... - * - */ - QueueSender queueSender = null; TestEventListener listener = new TestEventListener(); listener.setCurrentTimeMillis(START_TIME); - if (isLocal) { - queueSender = listener; - } else { - reportingBootstrapper = new ReportingBootstrapper(); - reportingBootstrapper.setOverriddenInstanceEventListener(listener); - reportingBootstrapper.start(); - queueSender = QueueFactory.getInstance().getSender(QueueIdentifier.INSTANCE); - } - System.out.println(" ----> GENERATING FALSE DATA " + (isLocal ? "(local)" : "(remote)")); + reportingBootstrapper = new ReportingBootstrapper(); + reportingBootstrapper.setOverriddenInstanceEventListener(listener); + reportingBootstrapper.start(); + + QueueSender queueSender = QueueFactory.getInstance().getSender(QueueIdentifier.INSTANCE); + List fakeInstances = new ArrayList(); From ff59af2f7803817274e44e7f4a1f1ee171bfb590 Mon Sep 17 00:00:00 2001 From: twerges Date: Tue, 15 Mar 2011 15:08:01 -0700 Subject: [PATCH 17/31] Corrected method call and removed superfluous method and interface from TestEventListener --- .../reporting/instance/FalseDataGenerator.java | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/clc/modules/reporting/src/test/java/com/eucalyptus/reporting/instance/FalseDataGenerator.java b/clc/modules/reporting/src/test/java/com/eucalyptus/reporting/instance/FalseDataGenerator.java index 1ca6a8b2ce1..d860301ac93 100644 --- a/clc/modules/reporting/src/test/java/com/eucalyptus/reporting/instance/FalseDataGenerator.java +++ b/clc/modules/reporting/src/test/java/com/eucalyptus/reporting/instance/FalseDataGenerator.java @@ -136,7 +136,7 @@ public static void runTest() removeFalseData(); printFalseData(); Thread.sleep(100000); - generateFalseData("remote"); + generateFalseData(); Thread.sleep(100000); printFalseData(); removeFalseData(); @@ -211,7 +211,6 @@ public static void summarizeFalseDataTwoCriteria( */ private static class TestEventListener extends InstanceEventListener - implements QueueSender { private long fakeCurrentTimeMillis = 0l; @@ -224,12 +223,6 @@ protected long getCurrentTimeMillis() { return fakeCurrentTimeMillis; } - - @Override - public void send(Event e) - { - super.fireEvent(e); - } } public static void main(String[] args) throws Exception From 028b831fbfb1bf16bfb897b4c93cbb06b5c78953 Mon Sep 17 00:00:00 2001 From: twerges Date: Tue, 15 Mar 2011 15:57:17 -0700 Subject: [PATCH 18/31] no longer need to start up QueueBroker in this way --- .../main/java/com/eucalyptus/bootstrap/SystemBootstrapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clc/modules/msgs/src/main/java/com/eucalyptus/bootstrap/SystemBootstrapper.java b/clc/modules/msgs/src/main/java/com/eucalyptus/bootstrap/SystemBootstrapper.java index b2012c91fa8..e103af5b186 100644 --- a/clc/modules/msgs/src/main/java/com/eucalyptus/bootstrap/SystemBootstrapper.java +++ b/clc/modules/msgs/src/main/java/com/eucalyptus/bootstrap/SystemBootstrapper.java @@ -130,7 +130,7 @@ public boolean init( ) throws Exception { * ActiveMQ broker from working properly. */ try { - GroovyUtil.eval("com.eucalyptus.reporting.queue.QueueBroker.getInstance().startup()"); + //GroovyUtil.eval("com.eucalyptus.reporting.queue.QueueBroker.getInstance().startup()"); LOG.info("Groovy eval of queue factory startup succeeded."); } catch (Exception ex) { LOG.error("Groovy eval of queue factory startup failed.", ex); From 69a4783311a3a4741e03d895a14a155a66ddbe34 Mon Sep 17 00:00:00 2001 From: twerges Date: Tue, 15 Mar 2011 15:57:46 -0700 Subject: [PATCH 19/31] added Queue subdirectory --- .../msgs/src/main/java/com/eucalyptus/system/SubDirectory.java | 1 + 1 file changed, 1 insertion(+) diff --git a/clc/modules/msgs/src/main/java/com/eucalyptus/system/SubDirectory.java b/clc/modules/msgs/src/main/java/com/eucalyptus/system/SubDirectory.java index 61af4eee3a6..294b4244a42 100644 --- a/clc/modules/msgs/src/main/java/com/eucalyptus/system/SubDirectory.java +++ b/clc/modules/msgs/src/main/java/com/eucalyptus/system/SubDirectory.java @@ -83,6 +83,7 @@ public enum SubDirectory { UPGRADE( BaseDirectory.CONF, "upgrade" ), REPORTS( BaseDirectory.CONF, "reports" ), CONF( BaseDirectory.CONF, "conf" ), + QUEUE( BaseDirectory.VAR, "queue" ), LIB( BaseDirectory.HOME, "/usr/share/eucalyptus" ); private static Logger LOG = Logger.getLogger( SubDirectory.class ); BaseDirectory parent; From 23b862203817a41f6f441748a9a441f5b9abfaff Mon Sep 17 00:00:00 2001 From: twerges Date: Tue, 15 Mar 2011 15:59:06 -0700 Subject: [PATCH 20/31] added method setOverrideenInstanceEventListener for testing --- .../reporting/ReportingBootstrapper.java | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/ReportingBootstrapper.java b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/ReportingBootstrapper.java index 21535f9cc51..d5918d4b81d 100644 --- a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/ReportingBootstrapper.java +++ b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/ReportingBootstrapper.java @@ -1,7 +1,6 @@ package com.eucalyptus.reporting; -import java.util.Timer; -import java.util.TimerTask; +import java.util.*; import org.apache.log4j.*; @@ -9,6 +8,7 @@ import com.eucalyptus.bootstrap.Bootstrapper; import com.eucalyptus.bootstrap.Provides; import com.eucalyptus.bootstrap.RunDuring; +import com.eucalyptus.component.*; import com.eucalyptus.component.id.Reporting; import com.eucalyptus.reporting.instance.*; import com.eucalyptus.reporting.storage.*; @@ -32,6 +32,7 @@ public class ReportingBootstrapper public ReportingBootstrapper() { + this.instanceListener = null; } @Override @@ -75,12 +76,15 @@ public boolean load() public boolean start() { try { - //TODO: brokers must FIND EACH OTHER here... + /* Start queue broker */ + QueueBroker.getInstance().startup(); + log.info("Queue broker started"); + queueFactory = QueueFactory.getInstance(); - //QueueFactory has been started in SystemBootstrapper.init() + queueFactory.startup(); /* Start storage receiver and storage queue poller thread */ @@ -101,7 +105,12 @@ public void run() */ QueueReceiver instanceReceiver = queueFactory.getReceiver(QueueIdentifier.INSTANCE); - instanceListener = new InstanceEventListener(); + if (instanceListener == null) { + instanceListener = new InstanceEventListener(); + log.info("New instance listener instantiated"); + } else { + log.info("Used existing instance listener"); + } instanceReceiver.addEventListener(instanceListener); log.info("ReportingBootstrapper started"); @@ -142,4 +151,10 @@ public boolean stop() } } + public void setOverriddenInstanceEventListener( + InstanceEventListener overriddenListener) + { + this.instanceListener = overriddenListener; + } + } From 9662a2263b2f133fa65eb191be1f4144ee813614 Mon Sep 17 00:00:00 2001 From: twerges Date: Tue, 15 Mar 2011 15:59:38 -0700 Subject: [PATCH 21/31] formatting --- .../instance/InstanceEventListener.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceEventListener.java b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceEventListener.java index 17174b1f7ea..d63dbea3368 100644 --- a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceEventListener.java +++ b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceEventListener.java @@ -2,7 +2,6 @@ import java.util.*; -import org.hibernate.*; import org.apache.log4j.*; import com.eucalyptus.entities.EntityWrapper; @@ -56,17 +55,18 @@ public void fireEvent( Event e ) * already. */ if (! recentlySeenUuids.contains(uuid)) { - try { - entityWrapper.getUnique( new InstanceAttributes( ) { - { - setUuid( uuid ); - } - } ); - } catch ( Exception ex ) { - entityWrapper.add(insAttrs); - log.info("Wrote Reporting Instance:" + uuid); - } - recentlySeenUuids.add(uuid); + try { + entityWrapper.getUnique(new InstanceAttributes() + { + { + setUuid(uuid); + } + }); + } catch (Exception ex) { + entityWrapper.add(insAttrs); + log.info("Wrote Reporting Instance:" + uuid); + } + recentlySeenUuids.add(uuid); } /* Gather all usage snapshots, and write them all to the database From d46c4ab035b2dea82176aaeaef2cc87266a1d1c1 Mon Sep 17 00:00:00 2001 From: twerges Date: Tue, 15 Mar 2011 16:00:07 -0700 Subject: [PATCH 22/31] javadoc changes --- .../reporting/instance/InstanceUsageLog.java | 34 +++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceUsageLog.java b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceUsageLog.java index 2fe53f59736..2577b255838 100644 --- a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceUsageLog.java +++ b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceUsageLog.java @@ -20,8 +20,8 @@ *

The data in the log is not indexed, in order to minimize write time. * As a result, you can't search through the log for specific instances or * specific data; you can only get a full dump between dates. However, there - * exist groovy scripts to filter the data, or to transfer the data into a - * data warehouse for subsequent searching. + * exist groovy scripts to transfer the data into a data warehouse for + * subsequent searching. * *

The usage data in logs is sampled, meaning data is collected * every n seconds and written. As a result, some small error will @@ -57,14 +57,18 @@ public static synchronized InstanceUsageLog getInstanceUsageLog() } /** - *

Scans through the usage log, then summarizes and groups the data it finds. - * For example, if you provide the criterion of "user", in scans through the usage - * log and adds up all usage data for each user, then returns a Map of - * UserId:UsageSummary. + *

Scans through the usage log, then summarizes the data it finds. + * Returns a map of instance usage, keyed by the value of the criterion you + * specified. For example, if you provide the criterion of "user", it scans + * through the usage log and adds up all usage data for each user, then + * returns a Map of UserId (String) -> UsageSummary. + * + *

This method is used for generating visual reports. For example, it + * could be used to generate a report of user id's in a left column with + * usage stats in various right-hand cols. * * @return A summary of all usage and instance data for a given criterion - * keyed by the criterion value. For example, a summary of all instance - * and usage data for each user, keyed by user. + * keyed by the criterion value. */ public Map scanSummarize(Period period, GroupByCriterion criterion) @@ -96,7 +100,10 @@ public Map scanSummarize(Period period, *

Scans through the usage log, then summarizes and groups the data it finds. * For example, scans through the usage log and adds up all usage data * for each user, within each Availability Zone, then returns the results - * as 

AvailZone->UserId->UsageSummary
. + * as 
AvailZoneId->UserId->UsageSummary
. + * + *

This is used to generate nested reports in which we list (for example) + * usage by user, grouped by availability zone. * * @return A summary of all usage and instance data for given criteria * keyed by the criterion values. @@ -152,6 +159,12 @@ private static String getAttributeValue(GroupByCriterion criterion, InstanceAttr } } + /** + *

LogScanResult is conceptually similar to a ResultSet. It contains the + * instance data and usage data for a particular period. + * + * @author tom.werges + */ public class LogScanResult { private final InstanceAttributes insAttrs; @@ -183,7 +196,8 @@ public UsageData getUsageData() } /** - *

Scans log for all results during some period. + *

Scans log for all results during some period. This method is used + * internally by the scanSummarize methods. */ public List scanLog(Period period) { From 0bf7432e48b1f4ec8002317d214068cc98a1a7fe Mon Sep 17 00:00:00 2001 From: twerges Date: Tue, 15 Mar 2011 16:00:31 -0700 Subject: [PATCH 23/31] minor javadoc chg --- .../eucalyptus/reporting/instance/InstanceUsageSnapshot.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceUsageSnapshot.java b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceUsageSnapshot.java index bb51680b8d8..dc1ca16c000 100644 --- a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceUsageSnapshot.java +++ b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceUsageSnapshot.java @@ -6,8 +6,7 @@ *

InstanceUsageSnapshot is a record of cumulative instance resource usage * at a certain point in time, keyed by instance uuid. * - *

InstanceUsageSnapshot allows null values for some of its fields. Null values - * indicate unknown usage and not zero usage. + *

InstanceUsageSnapshot allows null values for some of its fields. * * @author tom.werges */ From d844d739f8b36a44eeb1a3dbd6f8ba70307c4506 Mon Sep 17 00:00:00 2001 From: twerges Date: Tue, 15 Mar 2011 16:00:59 -0700 Subject: [PATCH 24/31] minor javadoc chg --- .../reporting/instance/InstanceUsageSummary.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceUsageSummary.java b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceUsageSummary.java index b925de4027f..7ee6683b76e 100644 --- a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceUsageSummary.java +++ b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceUsageSummary.java @@ -3,12 +3,10 @@ import com.eucalyptus.reporting.Period; /** - *

UsageSummary is a summary of all resource usage for some entity over a - * single time period. An entity can be a user, a group, an account, an - * availability zone, a cluster, and so on. UsageSummary includes instance - * types, instance running times, and resource usage. UsageSummary includes - * everything in a single bean which is necessary to generate a jasper - * report of instance usage. + *

InstanceUsageSummary is a summary of all resource usage for some entity + * over a time period. It includes summations of the instance types, + * numbers, running times, resource usage, etc, for a given user, + * account or other entity. * * @author tom.werges */ From 47c16b32f18340cbb721667241a7765bfd1f267a Mon Sep 17 00:00:00 2001 From: twerges Date: Tue, 15 Mar 2011 16:01:27 -0700 Subject: [PATCH 25/31] minor javadoc change --- .../main/java/com/eucalyptus/reporting/instance/UsageData.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/UsageData.java b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/UsageData.java index 72516d87abd..3f2b8dfdd99 100644 --- a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/UsageData.java +++ b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/UsageData.java @@ -2,7 +2,7 @@ /** *

UsageData represents usage of resources (like disk bandwidth, etc) - * by an instance over some period. UsageData is immutable. + * by an instance over a single period. UsageData is immutable. * *

Null values for a field indicate unknown usage, not zero usage. * From dad9f0fbf7ef71a53edf46ba84fcc0e16a74dd24 Mon Sep 17 00:00:00 2001 From: twerges Date: Tue, 15 Mar 2011 16:01:45 -0700 Subject: [PATCH 26/31] minor javadoc chg --- .../java/com/eucalyptus/reporting/instance/UsageSnapshot.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/UsageSnapshot.java b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/UsageSnapshot.java index eea41194263..292214c3ed7 100644 --- a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/UsageSnapshot.java +++ b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/UsageSnapshot.java @@ -1,7 +1,7 @@ package com.eucalyptus.reporting.instance; /** - * Snapshot of the resource usage of some instance at some point in time. + * UsageSnapshot of the resource usage of some instance at some point in time. * Contains cumulative usage data so it's populated with * all resource usage which has occurred up until this snapshot was * instantiated. From 64eec1bd1ddcd712069731e58778470272983cbc Mon Sep 17 00:00:00 2001 From: twerges Date: Tue, 15 Mar 2011 16:02:36 -0700 Subject: [PATCH 27/31] modified to find remote broker on the network if not on the same machine as reporting component. also changed static default strings --- .../reporting/queue/QueueBroker.java | 46 +++++++++++++------ 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/queue/QueueBroker.java b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/queue/QueueBroker.java index c9eed79f584..656dc838b7d 100644 --- a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/queue/QueueBroker.java +++ b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/queue/QueueBroker.java @@ -1,40 +1,42 @@ package com.eucalyptus.reporting.queue; -import java.util.ArrayList; -import java.util.List; +import java.util.*; + import org.apache.activemq.broker.BrokerService; import org.apache.activemq.command.ActiveMQDestination; import org.apache.log4j.*; +import com.eucalyptus.component.*; +import com.eucalyptus.component.id.Reporting; +import com.eucalyptus.system.SubDirectory; + public class QueueBroker { private static Logger log = Logger.getLogger( QueueBroker.class ); public static final String DEFAULT_NAME = "reportingBroker"; - public static final String DEFAULT_DIR = "/tmp"; - public static final String DEFAULT_URL = "tcp://localhost:63636"; + public static final String DEFAULT_DIR = SubDirectory.QUEUE.toString(); + public static final int DEFAULT_PORT = 63636; + public static final String DEFAULT_URL = "tcp://localhost:" + DEFAULT_PORT; + public static final String DEFAULT_REMOTE_URL_FORMAT = "static:(tcp://%s:%d)"; private boolean started = false; private String brokerName; private String brokerDataDir; private String brokerUrl; - //static:(tcp://remoteServer:63636) - private String remoteBrokerUrl; private List destinations; private BrokerService brokerService; private JmsBrokerThread brokerThread; - private QueueBroker(String brokerName, String brokerUrl, String remoteBrokerUrl, - String brokerDataDir) + private QueueBroker(String brokerName, String brokerUrl, String brokerDataDir) { this.brokerName = brokerName; this.brokerUrl = brokerUrl; this.brokerDataDir = brokerDataDir; this.destinations = new ArrayList(); - this.remoteBrokerUrl = remoteBrokerUrl; } private static QueueBroker instance; @@ -42,7 +44,7 @@ private QueueBroker(String brokerName, String brokerUrl, String remoteBrokerUrl, public static QueueBroker getInstance() { if (instance == null) { - return instance = new QueueBroker(DEFAULT_NAME, DEFAULT_URL, null, + return instance = new QueueBroker(DEFAULT_NAME, DEFAULT_URL, DEFAULT_DIR); } return instance; @@ -57,9 +59,23 @@ public void addDestination(String destName) public void startup() { - /* TODO: Determine our current location. - * Determine where the main reporting component lives. - * Establish a network connector. + /* Find remote broker if we're not running on the reporting machine + */ + String remoteBrokerUrl = null; + Component reportingComponent = Components.lookup(Reporting.class); + if (null!=reportingComponent && !reportingComponent.isLocal()) { + log.info("Searching for remote reporting broker"); + NavigableSet services = reportingComponent.getServices(); + for (Service service: services) { + remoteBrokerUrl = String.format(DEFAULT_REMOTE_URL_FORMAT, + service.getHost(), DEFAULT_PORT); + } + } else { + log.info("Reporting broker will run locally"); + remoteBrokerUrl = null; + } + + /* Startup BrokerService in separate thread and provide url and remoteUrl */ try { brokerService = new BrokerService(); @@ -69,7 +85,7 @@ public void startup() if (remoteBrokerUrl != null) { brokerService.addNetworkConnector(remoteBrokerUrl); } - brokerService.setUseJmx(true); + brokerService.setUseJmx(false); brokerThread = new JmsBrokerThread(brokerService); brokerThread.start(); Thread.sleep(1000); // give the broker a moment to startup; TODO: @@ -111,7 +127,7 @@ public static void main(String[] args) remoteBrokerUrl = args[0]; } QueueBroker broker = new QueueBroker(DEFAULT_NAME, DEFAULT_URL, - remoteBrokerUrl, DEFAULT_DIR); + DEFAULT_DIR); broker.startup(); } From 07aa8fce18364e1f697fab5ce78f85e215ee411d Mon Sep 17 00:00:00 2001 From: twerges Date: Tue, 15 Mar 2011 16:03:59 -0700 Subject: [PATCH 28/31] changed memory settings for JVM during test so it doesnt run out of memory --- clc/tools/runTest.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clc/tools/runTest.sh b/clc/tools/runTest.sh index a970970aadf..f783f074dbe 100755 --- a/clc/tools/runTest.sh +++ b/clc/tools/runTest.sh @@ -29,9 +29,9 @@ FILES=$(\ls -1 ${EUCALYPTUS}/usr/share/eucalyptus/*.jar) for FILE in $FILES; do export CLASSPATH=${FILE}:${CLASSPATH} done -CLASSPATH=${EUCALYPTUS}/etc/eucalyptus/cloud.d/upgrade:${EUCALYPTUS}/etc/eucalyptus/cloud.d/scripts:${CLASSPATH} +CLuSSPATH=${EUCALYPTUS}/etc/eucalyptus/cloud.d/upgrade:${EUCALYPTUS}/etc/eucalyptus/cloud.d/scripts:${CLASSPATH} echo -e "${CLASSPATH//:/\n}" -java -Xbootclasspath/p:${EUCALYPTUS}/usr/share/eucalyptus/openjdk-crypto.jar -classpath ${CLASSPATH} \ +java -Xms1g -Xmx3g -XX:MaxPermSize=768m -Xbootclasspath/p:${EUCALYPTUS}/usr/share/eucalyptus/openjdk-crypto.jar -classpath ${CLASSPATH} \ -Deuca.home=${EUCALYPTUS} \ -Deuca.lib.dir=${EUCALYPTUS} \ -Deuca.upgrade.new.dir=${EUCALYPTUS} \ From 48bcda072222a732202fcab888cd54f02aa44413 Mon Sep 17 00:00:00 2001 From: twerges Date: Tue, 15 Mar 2011 16:04:54 -0700 Subject: [PATCH 29/31] chris suggested do- prefix mistaken; worked --- clc/modules/module-inc.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clc/modules/module-inc.xml b/clc/modules/module-inc.xml index 2a910e62a61..06c0745bd43 100644 --- a/clc/modules/module-inc.xml +++ b/clc/modules/module-inc.xml @@ -178,7 +178,7 @@ - + From a90691953215ea5f75eb62c4eff4d9859908a043 Mon Sep 17 00:00:00 2001 From: twerges Date: Tue, 15 Mar 2011 16:05:47 -0700 Subject: [PATCH 30/31] build-test no longer calls bind; build-test-nogwt no longer calls ldap module --- clc/build.xml | 2 -- 1 file changed, 2 deletions(-) diff --git a/clc/build.xml b/clc/build.xml index 740d5c17711..1d9ead2b4bb 100644 --- a/clc/build.xml +++ b/clc/build.xml @@ -119,7 +119,6 @@ - @@ -129,7 +128,6 @@ - From e8ba4811e33e1befcafdd4df415d1c38200a5b78 Mon Sep 17 00:00:00 2001 From: twerges Date: Tue, 15 Mar 2011 16:06:46 -0700 Subject: [PATCH 31/31] added reporting test to classpath --- clc/.classpath | 1 + 1 file changed, 1 insertion(+) diff --git a/clc/.classpath b/clc/.classpath index e5bb398133f..896a5039d4e 100644 --- a/clc/.classpath +++ b/clc/.classpath @@ -23,6 +23,7 @@ +