From e43c5b7f62e35d095d7f3ecfe6239fe10666d8cf Mon Sep 17 00:00:00 2001 From: Fabrice Bacchella Date: Tue, 30 Aug 2016 17:58:35 +0200 Subject: [PATCH] Downloading data sources was failing because the way datasources are populated was different for a RrdGraphDef or a DataProcessor. Added a generic classes that abstract that. --- src/jrds/DatasourcesPopulator.java | 190 +++++++++++++++++++++++++ src/jrds/GraphDesc.java | 215 ++++------------------------- 2 files changed, 214 insertions(+), 191 deletions(-) create mode 100644 src/jrds/DatasourcesPopulator.java diff --git a/src/jrds/DatasourcesPopulator.java b/src/jrds/DatasourcesPopulator.java new file mode 100644 index 000000000..561a43677 --- /dev/null +++ b/src/jrds/DatasourcesPopulator.java @@ -0,0 +1,190 @@ +package jrds; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.log4j.Logger; +import org.rrd4j.data.DataProcessor; +import org.rrd4j.data.Plottable; +import org.rrd4j.data.Variable; +import org.rrd4j.graph.RrdGraphDef; + +import jrds.GraphDesc.DsDesc; +import jrds.GraphDesc.GraphType; +import jrds.store.ExtractInfo; +import jrds.store.Extractor; + +import static jrds.Util.delayedFormatString; + +/** + * A class that populate a DataProcess or a RrdGraphDef with data from a probe and custom data + * @author Fabrice Bacchella + * + * @param + */ +class DatasourcesPopulator { + + static final private Logger logger = Logger.getLogger(DatasourcesPopulator.class); + + final RrdGraphDef graphDef; + final DataProcessor dp; + + private final List toDo = new ArrayList(); + + static List populate(RrdGraphDef graphDef, Probe defProbe, ExtractInfo ei, Map customData, List allds, String name) { + DatasourcesPopulator p = new DatasourcesPopulator(graphDef, defProbe, ei, customData, allds, name); + return Collections.unmodifiableList(p.toDo); + } + + static DataProcessor populate(Probe defProbe, ExtractInfo ei, Map customData, List allds, String name) { + DataProcessor dp = ei.getDataProcessor(); + new DatasourcesPopulator(dp, defProbe, ei, customData, allds, name); + return dp; + } + + private DatasourcesPopulator(T wrapped, Probe defProbe, ExtractInfo ei, Map customData, List allds, String name) { + HostsList hl = defProbe.getHostList(); + + if (wrapped instanceof RrdGraphDef) { + graphDef = (RrdGraphDef) wrapped; + dp = null; + } else if (wrapped instanceof DataProcessor) { + dp = (DataProcessor) wrapped; + graphDef = null; + } else { + throw new RuntimeException(); + } + + // The datasources already found + Set datasources = new HashSet(); + + // The needed extractors + Map, Extractor> probeDS = new HashMap, Extractor>(1); + probeDS.put(defProbe, defProbe.getMainStore().getExtractor()); + + for (DsDesc ds: allds) { + boolean complete; + // Not a data source, don't try to add it in datasources + if (!ds.graphType.datasource()) { + complete = true; + } + // The graph is a percentile + else if (ds.percentile != null) { + complete = true; + if (!datasources.contains(ds.name)) { + percentile(ds.name, ds.dsName, ds.percentile); + datasources.add(ds.name); + } + } + // A rpn datasource + else if(ds.rpn != null) { + complete = true; + if(!datasources.contains(ds.name)) { + datasource(ds.name, ds.rpn); + datasources.add(ds.name); + } + } + // A legend + else if(ds.graphType == GraphType.LEGEND) { + complete = true; + } + // Does the datas existe in the provided values + // It override existing values in the probe + else if(customData != null && customData.containsKey(ds.dsName)) { + complete = true; + if(!datasources.contains(ds.name)) { + datasource(ds.name, customData.get(ds.dsName)); + datasources.add(ds.name); + logger.trace(delayedFormatString("custom data found for %s", ds.dsName)); + } + } + // Last but common case, datasource refers to a rrd + // Or they might be on the associated rrd + else { + Probe probe = defProbe; + if(ds.dspath != null) { + // If the host is not defined, use the current host + String pathHost = ds.dspath.host; + if(pathHost == null) { + pathHost = defProbe.getHost().getName(); + } + logger.trace(delayedFormatString("External probe path: %s/%s/%s", pathHost, ds.dspath.probe, ds.dsName)); + probe = hl.getProbeByPath(pathHost, ds.dspath.probe); + if(probe == null) { + logger.error("Invalide probe: " + pathHost + "/" + ds.dspath.probe); + continue; + } + } + if(!probe.dsExist(ds.dsName)) { + logger.error("Invalide datasource " + ds.dsName + ", not found in " + probe); + continue; + } + complete = true; + + // Add the dsName for the probe found + if(!probeDS.containsKey(probe)) { + probeDS.put(probe, probe.getMainStore().getExtractor()); + } + Extractor ex = probeDS.get(probe); + if(!datasources.contains(ds.name)) { + ex.addSource(ds.name, ds.dsName); + datasources.add(ds.name); + } else { + logger.error("Datasource '" + ds.dsName + "' defined twice in " + name + ", for found: " + ds); + } + } + if(complete) { + toDo.add(ds); + } else { + logger.debug("Error for " + ds); + logger.error("No way to plot " + ds.name + " in " + name + " found"); + } + } + + // Fill the graphdef with extracted data + for(Extractor x: probeDS.values()) { + if (graphDef != null) { + x.fill(graphDef, ei); + } else { + x.fill(dp, ei); + } + x.release(); + } + + logger.trace(delayedFormatString("Datasource: %s", datasources)); + if (graphDef != null) { + logger.trace(delayedFormatString("Todo: : %s", toDo)); + } + + } + + private void datasource(String name, Plottable plottable) { + if (graphDef != null) { + graphDef.datasource(name, plottable); + } else { + dp.addDatasource(name, plottable); + } + } + + private void datasource(String name, String rpn) { + if (graphDef != null) { + graphDef.datasource(name, rpn); + } else { + dp.addDatasource(name, rpn); + } + } + + private void percentile(String name, String dsName, int percentile) { + if (graphDef != null) { + graphDef.datasource(name, dsName, new Variable.PERCENTILE(percentile)); + } else { + dp.addDatasource(name, dsName, new Variable.PERCENTILE(percentile)); + } + } + +} diff --git a/src/jrds/GraphDesc.java b/src/jrds/GraphDesc.java index f3133bde8..335154abf 100644 --- a/src/jrds/GraphDesc.java +++ b/src/jrds/GraphDesc.java @@ -8,34 +8,32 @@ import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; -import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Set; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; -import jrds.Util.SiPrefix; -import jrds.probe.IndexedProbe; -import jrds.probe.UrlProbe; -import jrds.probe.jdbc.JdbcProbe; -import jrds.store.ExtractInfo; -import jrds.store.Extractor; -import jrds.webapp.ACL; -import jrds.webapp.WithACL; - import org.apache.log4j.Logger; import org.rrd4j.ConsolFun; import org.rrd4j.data.DataProcessor; import org.rrd4j.data.Plottable; +import org.rrd4j.data.Variable; import org.rrd4j.graph.RrdGraphConstants; import org.rrd4j.graph.RrdGraphDef; import org.w3c.dom.Document; import org.w3c.dom.Element; +import jrds.Util.SiPrefix; +import jrds.probe.IndexedProbe; +import jrds.probe.UrlProbe; +import jrds.probe.jdbc.JdbcProbe; +import jrds.store.ExtractInfo; +import jrds.webapp.ACL; +import jrds.webapp.WithACL; + /** * A classed used to store the static description of a graph * @@ -520,7 +518,7 @@ public static final Color resolveIndex(int i) { } } - static private final class DsDesc { + static final class DsDesc { final String name; final String dsName; final String rpn; @@ -820,100 +818,7 @@ public RrdGraphDef getEmptyGraphDef() { * associated probe */ public void fillGraphDef(RrdGraphDef graphDef, Probe defProbe, ExtractInfo ei, Map customData) { - HostsList hl = defProbe.getHostList(); - List toDo = new ArrayList(); - // The datasources already found - Set datasources = new HashSet(); - - // The needed extractors - Map, Extractor> probeDS = new HashMap, Extractor>(1); - probeDS.put(defProbe, defProbe.getMainStore().getExtractor()); - - for(DsDesc ds: allds) { - boolean complete; - // Not a data source, don't try to add it in datasources - if(!ds.graphType.datasource()) { - complete = true; - } - // The graph is a percentile - else if(ds.percentile != null) { - complete = true; - if(!datasources.contains(ds.name)) { - graphDef.percentile(ds.name, ds.dsName, ds.percentile); - datasources.add(ds.name); - } - } - // A rpn datasource - else if(ds.rpn != null) { - complete = true; - if(!datasources.contains(ds.name)) { - graphDef.datasource(ds.name, ds.rpn); - datasources.add(ds.name); - } - } - // A legend - else if(ds.graphType == GraphType.LEGEND) { - complete = true; - } - // Does the datas existe in the provided values - // It override existing values in the probe - else if(customData != null && customData.containsKey(ds.dsName)) { - complete = true; - if(!datasources.contains(ds.name)) { - graphDef.datasource(ds.name, customData.get(ds.dsName)); - datasources.add(ds.name); - logger.trace(Util.delayedFormatString("custom data found for %s", ds.dsName)); - } - } - // Last but common case, datasource refers to a rrd - // Or they might be on the associated rrd - else { - Probe probe = defProbe; - if(ds.dspath != null) { - // If the host is not defined, use the current host - String pathHost = ds.dspath.host; - if(pathHost == null) { - pathHost = defProbe.getHost().getName(); - } - logger.trace(jrds.Util.delayedFormatString("External probe path: %s/%s/%s", pathHost, ds.dspath.probe, ds.dsName)); - probe = hl.getProbeByPath(pathHost, ds.dspath.probe); - if(probe == null) { - logger.error("Invalide probe: " + pathHost + "/" + ds.dspath.probe); - continue; - } - } - if(!probe.dsExist(ds.dsName)) { - logger.error("Invalide datasource " + ds.dsName + ", not found in " + probe); - continue; - } - complete = true; - - // Add the dsName for the probe found - if(!probeDS.containsKey(probe)) { - probeDS.put(probe, probe.getMainStore().getExtractor()); - } - Extractor ex = probeDS.get(probe); - if(!datasources.contains(ds.name)) { - ex.addSource(ds.name, ds.dsName); - datasources.add(ds.name); - } else { - logger.error("Datasource '" + ds.dsName + "' defined twice in " + name + ", for found: " + ds); - } - } - if(complete) { - toDo.add(ds); - } else { - logger.debug("Error for " + ds); - logger.error("No way to plot " + ds.name + " in " + name + " found"); - } - } - - // Fill the graphdef with extracted data - for(Extractor x: probeDS.values()) { - x.fill(graphDef, ei); - x.release(); - } - + List toDo = DatasourcesPopulator.populate(graphDef, defProbe, ei, customData, allds, name); // The title line, only if values block is required if(withSummary) { graphDef.comment(""); // We simulate the color box @@ -925,9 +830,6 @@ else if(customData != null && customData.containsKey(ds.dsName)) { graphDef.comment("\\l"); } - logger.trace(Util.delayedFormatString("Datasource: %s", datasources)); - logger.trace(Util.delayedFormatString("Todo: : %s", toDo)); - String shortLegend = withSummary ? " \\g" : null; for(DsDesc ds: toDo) { ds.graphType.draw(graphDef, ds.name, ds.color, shortLegend); @@ -952,88 +854,6 @@ public RrdGraphDef getGraphDef(Probe defProbe, ExtractInfo ei, Map defProbe, ExtractInfo ei, Map customData) throws IOException { - DataProcessor retValue = ei.getDataProcessor(); - - HostsList hl = defProbe.getHostList(); - - // The datasources already found - Set datasources = new HashSet(); - - // The needed extractors - Map, Extractor> probeDS = new HashMap, Extractor>(1); - probeDS.put(defProbe, defProbe.getMainStore().getExtractor()); - - String lastName = null; - for(DsDesc ds: allds) { - boolean stack = ds.graphType == GraphType.STACK; - boolean plotted = stack || ds.graphType == GraphType.LINE || ds.graphType == GraphType.AREA; - if(ds.rpn == null && ds.dsName != null) { - // Does the datas exists in the provided values - if(customData != null && customData.containsKey(ds.dsName)) { - retValue.addDatasource(ds.name, customData.get(ds.dsName)); - } else { - Probe probe = defProbe; - if(ds.dspath != null) { - logger.trace(jrds.Util.delayedFormatString("External probe path: %s/%s/%s", ds.dspath.host, ds.dspath.probe, ds.dsName)); - probe = hl.getProbeByPath(ds.dspath.host, ds.dspath.probe); - if(probe == null) { - logger.error("Invalide probe: " + ds.dspath.host + "/" + ds.dspath.probe); - continue; - } - } - if(!probe.dsExist(ds.dsName)) { - logger.error("Invalide datasource " + ds.dsName + ", not found in " + probe); - continue; - } - logger.trace(Util.delayedFormatString("ds '%s' found in probe %s", ds.dsName, probe)); - // Add the dsName for the probe found - if(!probeDS.containsKey(probe)) { - probeDS.put(probe, probe.getMainStore().getExtractor()); - } - Extractor ex = probeDS.get(probe); - if(!datasources.contains(ds.dsName)) { - ex.addSource(ds.dsName, ds.name); - datasources.add(ds.dsName); - } else { - logger.error("Datasource '" + ds.dsName + "' defined twice in " + name); - } - } - } else if(ds.rpn != null) { - retValue.addDatasource(ds.name, ds.rpn); - datasources.add(ds.dsName); - } - - if(plotted && stack) { - retValue.addDatasource("Plotted" + ds.name, lastName + ", " + ds.name + ", +"); - } else if(plotted) { - retValue.addDatasource("Plotted" + ds.name, ds.name); - } - lastName = ds.name; - } - if(logger.isTraceEnabled()) { - logger.trace("Datasource for " + getName()); - for(String s: retValue.getSourceNames()) - logger.trace("\t" + s); - } - - // Fill the dataprocessor with extracted data - for(Extractor x: probeDS.values()) { - x.fill(retValue, ei); - x.release(); - } - return retValue; - } - protected void addLegend(RrdGraphDef def, String ds, GraphType gt, String legend) { if(legend == null) return; @@ -1059,6 +879,19 @@ protected void addLegend(RrdGraphDef def, String ds, GraphType gt, String legend } } + /** + * return the RrdGraphDef for this graph, used the indicated probe any data + * can be overridden of a provided map of Plottable + * + * @param probe + * @param ownData data used to override probe's own values + * @return + * @throws IOException + */ + public DataProcessor getPlottedDatas(Probe defProbe, ExtractInfo ei, Map customData) throws IOException { + return DatasourcesPopulator.populate(defProbe, ei, customData, allds, name); + } + /** * @return Returns the graphTitle. */