GenerateReport is a command-line tool for rapid testing which uses + * jasper reports internally. It takes command-line args from the + *
runTest.shmechanism and writes a resultant report to a file. + * + * @author tom.werges + */ public class GenerateReport { public static enum Format @@ -62,6 +69,8 @@ public String getFileExtension() } + /* Expects jrxml files to be in /tmp and writes results there. + */ private static File baseDir = new File("/tmp/"); private static File storageReportFile = new File(baseDir, "storage.jrxml"); private static File nestedStorageReportFile = new File(baseDir, "nested_storage.jrxml"); diff --git a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/ReportingBootstrapper.java b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/ReportingBootstrapper.java index 21535f9cc51..d5918d4b81d 100644 --- a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/ReportingBootstrapper.java +++ b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/ReportingBootstrapper.java @@ -1,7 +1,6 @@ package com.eucalyptus.reporting; -import java.util.Timer; -import java.util.TimerTask; +import java.util.*; import org.apache.log4j.*; @@ -9,6 +8,7 @@ import com.eucalyptus.bootstrap.Bootstrapper; import com.eucalyptus.bootstrap.Provides; import com.eucalyptus.bootstrap.RunDuring; +import com.eucalyptus.component.*; import com.eucalyptus.component.id.Reporting; import com.eucalyptus.reporting.instance.*; import com.eucalyptus.reporting.storage.*; @@ -32,6 +32,7 @@ public class ReportingBootstrapper public ReportingBootstrapper() { + this.instanceListener = null; } @Override @@ -75,12 +76,15 @@ public boolean load() public boolean start() { try { - //TODO: brokers must FIND EACH OTHER here... + /* Start queue broker */ + QueueBroker.getInstance().startup(); + log.info("Queue broker started"); + queueFactory = QueueFactory.getInstance(); - //QueueFactory has been started in SystemBootstrapper.init() + queueFactory.startup(); /* Start storage receiver and storage queue poller thread */ @@ -101,7 +105,12 @@ public void run() */ QueueReceiver instanceReceiver = queueFactory.getReceiver(QueueIdentifier.INSTANCE); - instanceListener = new InstanceEventListener(); + if (instanceListener == null) { + instanceListener = new InstanceEventListener(); + log.info("New instance listener instantiated"); + } else { + log.info("Used existing instance listener"); + } instanceReceiver.addEventListener(instanceListener); log.info("ReportingBootstrapper started"); @@ -142,4 +151,10 @@ public boolean stop() } } + public void setOverriddenInstanceEventListener( + InstanceEventListener overriddenListener) + { + this.instanceListener = overriddenListener; + } + } diff --git a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceDisplayBean.java b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceDisplayBean.java index 28e0802f987..426faef164d 100644 --- a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceDisplayBean.java +++ b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceDisplayBean.java @@ -97,5 +97,10 @@ public Long getDiskIoSize() return UnitUtil.convertSize(summary.getDiskIoMegs(), INTERNAL_UNITS.getSizeUnit(), units.getSizeUnit()); } + + public Units getUnits() + { + return units; + } } diff --git a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceEventListener.java b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceEventListener.java index 17174b1f7ea..d63dbea3368 100644 --- a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceEventListener.java +++ b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceEventListener.java @@ -2,7 +2,6 @@ import java.util.*; -import org.hibernate.*; import org.apache.log4j.*; import com.eucalyptus.entities.EntityWrapper; @@ -56,17 +55,18 @@ public void fireEvent( Event e ) * already. */ if (! recentlySeenUuids.contains(uuid)) { - try { - entityWrapper.getUnique( new InstanceAttributes( ) { - { - setUuid( uuid ); - } - } ); - } catch ( Exception ex ) { - entityWrapper.add(insAttrs); - log.info("Wrote Reporting Instance:" + uuid); - } - recentlySeenUuids.add(uuid); + try { + entityWrapper.getUnique(new InstanceAttributes() + { + { + setUuid(uuid); + } + }); + } catch (Exception ex) { + entityWrapper.add(insAttrs); + log.info("Wrote Reporting Instance:" + uuid); + } + recentlySeenUuids.add(uuid); } /* Gather all usage snapshots, and write them all to the database diff --git a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceUsageLog.java b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceUsageLog.java index 0a0f1e9fea8..2577b255838 100644 --- a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceUsageLog.java +++ b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceUsageLog.java @@ -6,6 +6,8 @@ import java.util.List; import java.util.Map; import org.apache.log4j.Logger; +import org.hibernate.Session; + import com.eucalyptus.configurable.ConfigurableClass; import com.eucalyptus.entities.EntityWrapper; import com.eucalyptus.reporting.GroupByCriterion; @@ -18,8 +20,8 @@ *
The data in the log is not indexed, in order to minimize write time. * As a result, you can't search through the log for specific instances or * specific data; you can only get a full dump between dates. However, there - * exist groovy scripts to filter the data, or to transfer the data into a - * data warehouse for subsequent searching. + * exist groovy scripts to transfer the data into a data warehouse for + * subsequent searching. * *
The usage data in logs is sampled, meaning data is collected * every n seconds and written. As a result, some small error will @@ -55,14 +57,18 @@ public static synchronized InstanceUsageLog getInstanceUsageLog() } /** - *
Scans through the usage log, then summarizes and groups the data it finds. - * For example, if you provide the criterion of "user", in scans through the usage - * log and adds up all usage data for each user, then returns a Map of - * UserId:UsageSummary. + *
Scans through the usage log, then summarizes the data it finds. + * Returns a map of instance usage, keyed by the value of the criterion you + * specified. For example, if you provide the criterion of "user", it scans + * through the usage log and adds up all usage data for each user, then + * returns a Map of UserId (String) -> UsageSummary. + * + *
This method is used for generating visual reports. For example, it
+ * could be used to generate a report of user id's in a left column with
+ * usage stats in various right-hand cols.
*
* @return A summary of all usage and instance data for a given criterion
- * keyed by the criterion value. For example, a summary of all instance
- * and usage data for each user, keyed by user.
+ * keyed by the criterion value.
*/
public Map Scans through the usage log, then summarizes and groups the data it finds.
* For example, scans through the usage log and adds up all usage data
* for each user, within each Availability Zone, then returns the results
- * as This is used to generate nested reports in which we list (for example)
+ * usage by user, grouped by availability zone.
*
* @return A summary of all usage and instance data for given criteria
* keyed by the criterion values.
@@ -150,6 +159,12 @@ private static String getAttributeValue(GroupByCriterion criterion, InstanceAttr
}
}
+ /**
+ * LogScanResult is conceptually similar to a ResultSet. It contains the
+ * instance data and usage data for a particular period.
+ *
+ * @author tom.werges
+ */
public class LogScanResult
{
private final InstanceAttributes insAttrs;
@@ -181,7 +196,8 @@ public UsageData getUsageData()
}
/**
- * Scans log for all results during some period.
+ * Scans log for all results during some period. This method is used
+ * internally by the scanSummarize methods.
*/
public List InstanceUsageSnapshot is a record of cumulative instance resource usage
* at a certain point in time, keyed by instance uuid.
*
- * InstanceUsageSnapshot allows null values for some of its fields. Null values
- * indicate unknown usage and not zero usage.
+ * InstanceUsageSnapshot allows null values for some of its fields.
*
* @author tom.werges
*/
diff --git a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceUsageSummary.java b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceUsageSummary.java
index b925de4027f..7ee6683b76e 100644
--- a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceUsageSummary.java
+++ b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/InstanceUsageSummary.java
@@ -3,12 +3,10 @@
import com.eucalyptus.reporting.Period;
/**
- * UsageSummary is a summary of all resource usage for some entity over a
- * single time period. An entity can be a user, a group, an account, an
- * availability zone, a cluster, and so on. UsageSummary includes instance
- * types, instance running times, and resource usage. UsageSummary includes
- * everything in a single bean which is necessary to generate a jasper
- * report of instance usage.
+ * InstanceUsageSummary is a summary of all resource usage for some entity
+ * over a time period. It includes summations of the instance types,
+ * numbers, running times, resource usage, etc, for a given user,
+ * account or other entity.
*
* @author tom.werges
*/
diff --git a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/UsageData.java b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/UsageData.java
index 72516d87abd..3f2b8dfdd99 100644
--- a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/UsageData.java
+++ b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/UsageData.java
@@ -2,7 +2,7 @@
/**
* UsageData represents usage of resources (like disk bandwidth, etc)
- * by an instance over some period. UsageData is immutable.
+ * by an instance over a single period. UsageData is immutable.
*
* Null values for a field indicate unknown usage, not zero usage.
*
diff --git a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/UsageSnapshot.java b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/UsageSnapshot.java
index eea41194263..292214c3ed7 100644
--- a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/UsageSnapshot.java
+++ b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/instance/UsageSnapshot.java
@@ -1,7 +1,7 @@
package com.eucalyptus.reporting.instance;
/**
- * Snapshot of the resource usage of some instance at some point in time.
+ * UsageSnapshot of the resource usage of some instance at some point in time.
* Contains cumulative usage data so it's populated with
* all resource usage which has occurred up until this snapshot was
* instantiated.
diff --git a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/queue/QueueBootstrapper.java b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/queue/QueueBootstrapper.java
index 9fc5f469c92..8116c64e196 100644
--- a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/queue/QueueBootstrapper.java
+++ b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/queue/QueueBootstrapper.java
@@ -15,14 +15,18 @@ public class QueueBootstrapper extends Bootstrapper
@Override
public boolean load() throws Exception
{
- try {
- QueueFactory.getInstance().startup();
- log.info("broker started");
- return true;
- } catch (Exception ex) {
- ex.printStackTrace();
- return false;
- }
+ /* NOTE: a workaround was moved to SystemBootstrapper.init() because
+ * log4j brain damage prevents ActiveMQ broker from starting.
+ */
+// try {
+// QueueFactory.getInstance().startup();
+// log.info("broker started");
+// return true;
+// } catch (Exception ex) {
+// ex.printStackTrace();
+// return false;
+// }
+ return true;
}
@Override
diff --git a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/queue/QueueBroker.java b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/queue/QueueBroker.java
index 4ec767766be..656dc838b7d 100644
--- a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/queue/QueueBroker.java
+++ b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/queue/QueueBroker.java
@@ -1,29 +1,25 @@
package com.eucalyptus.reporting.queue;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.*;
+
import org.apache.activemq.broker.BrokerService;
import org.apache.activemq.command.ActiveMQDestination;
import org.apache.log4j.*;
+import com.eucalyptus.component.*;
+import com.eucalyptus.component.id.Reporting;
+import com.eucalyptus.system.SubDirectory;
+
public class QueueBroker
{
private static Logger log = Logger.getLogger( QueueBroker.class );
public static final String DEFAULT_NAME = "reportingBroker";
- public static final String DEFAULT_DIR = "/tmp";
- public static final String DEFAULT_URL = "tcp://localhost:63636";
+ public static final String DEFAULT_DIR = SubDirectory.QUEUE.toString();
+ public static final int DEFAULT_PORT = 63636;
+ public static final String DEFAULT_URL = "tcp://localhost:" + DEFAULT_PORT;
+ public static final String DEFAULT_REMOTE_URL_FORMAT = "static:(tcp://%s:%d)";
- private static QueueBroker instance = null;
-
- public static QueueBroker getInstance()
- {
- if (instance == null) {
- instance = new QueueBroker(DEFAULT_NAME, DEFAULT_URL, DEFAULT_DIR);
- }
- return instance;
- }
-
private boolean started = false;
private String brokerName;
@@ -43,6 +39,17 @@ private QueueBroker(String brokerName, String brokerUrl, String brokerDataDir)
this.destinations = new ArrayList StorageEventPoller is run periodically and polls the message queue for
+ * new storage events. It then aggregates those events into snapshots.
+ *
+ * @author tom.werges
+ */
public class StorageEventPoller
{
private static Logger LOG = Logger.getLogger( StorageEventPoller.class );
@@ -26,6 +32,8 @@ public StorageEventPoller(QueueReceiver receiver)
public void writeEvents()
{
+ LOG.info("Queue poll:" + System.currentTimeMillis());
+
EntityWrapper StorageUsageLog
+ * StorageUsageLog is the main API for accessing storage usage information.
*
* @author tom.werges
*/
@@ -232,6 +232,7 @@ public Map StorageUsageSnapshot is a snapshot of disk data usage at some point in
* time.
*
- * @author twerges
+ * @author tom.werges
*/
@Entity
@PersistenceContext(name="reporting")
@@ -22,7 +22,7 @@ protected StorageUsageSnapshot()
{
}
-
+
public StorageUsageSnapshot(SnapshotKey key, StorageUsageData usageData)
{
this.key = key;
@@ -43,4 +43,5 @@ public String toString()
{
return String.format("[key:%s,usageData:%s]", this.key, this.usageData);
}
+
}
diff --git a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/storage/StorageUsageSummary.java b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/storage/StorageUsageSummary.java
index 3abd013e5d6..a0473634db3 100644
--- a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/storage/StorageUsageSummary.java
+++ b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/storage/StorageUsageSummary.java
@@ -6,7 +6,7 @@
* usage.
*
* StorageUsageSummary is not a bean. StorageUsageSummary accumulates
- * data through the The reporting.storage package contains classes for the storage and retrieval
+of historical usage information for Walrus/S3. It could be used to generate
+timely visual reports, of S3/Walrus usage over some past time period (like a
+month), in order to monitor usage trends or to conduct billing.
+
+ The reporting.storage package stores its accounting information as a series
+of snapshots, each of which contains the total resource usage for
+a given (owner,account,cluster,availability_zone) combination over some time
+period.
+
+ The reporting.storage package receives events from S3/Walrus indicating
+ allocation and deallocation of Walrus/S3 objects. Then the reporting.storage
+package uses those events to generate snapshots, every n seconds, of
+total resource utilization for each (owner,account,cluster,zone) combination.
+
+ Events are transmitted to the reporting.storage package over a reliable
+message queue. Events are transmitted using the reporting.queue package.
+
+ Reporting data stored by the reporting.storage package can be accessed
+using the In addition, there is a The reporting.storage package is used by FalseDataGenerator generates false data about instances.
- * It generates fake starting and ending times, imaginary resource
- * usage, fictitious clusters, and non-existent accounts and users.
+ * FalseDataGenerator generates false data about instances. It generates
+ * fake starting and ending times, imaginary resource usage, fictitious
+ * clusters, and non-existent accounts and users.
*
- * The main use of this class is to be called from the
- * FalseDataGenerator is meant to be called from the command-line tool,
+ * False data should be deleted afterward by calling the
* AvailZone->UserId->UsageSummary
.
+ * as AvailZoneId->UserId->UsageSummary
.
+ *
+ * update
method and then returns results of
+ * data through the update
method and then returns resulting
* statistics using the various get
methods.
*
* @author tom.werges
diff --git a/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/storage/package.html b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/storage/package.html
new file mode 100644
index 00000000000..90105d2c7ab
--- /dev/null
+++ b/clc/modules/reporting/src/main/java/com/eucalyptus/reporting/storage/package.html
@@ -0,0 +1,33 @@
+
+StorageUsageLog
class. This class will allow you to scan
+through the usage logs and gather usage information. This is the main class
+for accessing reporting.storage information.
+
+StorageDisplayDb
class which is built
+atop the StorageUsageLog
class. The StorageDisplayDb
+class adds logic for displayable Strings, unit conversions, and so on. It
+emits displayable bean classes which can be used by jasper or other report
+generation engines.
+
+Reports
class in the
+www
module, which generates visual reports based on user
+selections.
diff --git a/clc/modules/reporting/src/test/java/com/eucalyptus/reporting/instance/FalseDataGenerator.java b/clc/modules/reporting/src/test/java/com/eucalyptus/reporting/instance/FalseDataGenerator.java
index 5da01476996..92caefd4439 100644
--- a/clc/modules/reporting/src/test/java/com/eucalyptus/reporting/instance/FalseDataGenerator.java
+++ b/clc/modules/reporting/src/test/java/com/eucalyptus/reporting/instance/FalseDataGenerator.java
@@ -2,8 +2,10 @@
import java.util.*;
+import com.eucalyptus.reporting.ReportingBootstrapper;
import com.eucalyptus.reporting.GroupByCriterion;
import com.eucalyptus.reporting.Period;
+import com.eucalyptus.reporting.event.Event;
import com.eucalyptus.reporting.event.InstanceEvent;
import com.eucalyptus.reporting.queue.QueueBroker;
import com.eucalyptus.reporting.queue.QueueFactory;
@@ -12,13 +14,13 @@
import com.eucalyptus.upgrade.TestDescription;
/**
- * clc/tools/runTest.sh
tool, specifying the class name and a
- * method below.
+ * clc/tools/runTest.sh
, specifying the class name and a method
+ * below.
*
* deleteFalseData
method.
@@ -40,24 +42,31 @@ public class FalseDataGenerator
private static final int NUM_ACCOUNT = 16;
private static final int NUM_CLUSTER = 4;
private static final int NUM_AVAIL_ZONE = 2;
+
+ private static ReportingBootstrapper reportingBootstrapper = null;
private enum FalseInstanceType
{
M1SMALL, C1MEDIUM, M1LARGE, M1XLARGE, C1XLARGE;
}
- /**
- * @param remote Must equal "local" if this is to be run locally; any other
- * value means it will run over the MQ and will search for a broker.
- */
- public static void generateFalseData(String remote)
+ public static void generateFalseData()
{
- boolean isLocal = (remote != null && remote.equalsIgnoreCase("local"));
- System.out.println(" ----> GENERATING FALSE DATA " + (isLocal ? "(local)" : "(remote)"));
-
+ System.out.println(" ----> GENERATING FALSE DATA");
+
+ TestEventListener listener = new TestEventListener();
+ listener.setCurrentTimeMillis(START_TIME);
+
+ reportingBootstrapper = new ReportingBootstrapper();
+ reportingBootstrapper.setOverriddenInstanceEventListener(listener);
+ reportingBootstrapper.start();
+
+ QueueSender queueSender = QueueFactory.getInstance().getSender(QueueIdentifier.INSTANCE);
+
+
List