diff --git a/build.sh b/build.sh index e7ccba24..41c5f0e7 100755 --- a/build.sh +++ b/build.sh @@ -15,3 +15,4 @@ export IMAGE=${DOCKER_REGISTRY}/${IMAGE_NAME}:${IMAGE_VERSION} mvn clean package -DskipTests docker build -t=$IMAGE . +docker push $IMAGE diff --git a/examples/metrics/default-metrics.toml b/examples/metrics/default-metrics.toml new file mode 100644 index 00000000..b7d8f7ff --- /dev/null +++ b/examples/metrics/default-metrics.toml @@ -0,0 +1,6 @@ +[[metric]] +context = "sessions" +labels = ["inst_id", "status", "type"] +metricsdesc = { value = "Gauge metric with count of sessions by status and type." } +request = "select inst_id, status, type, count(*) as value from gv$session group by status, type, inst_id" +ignorezeroresult = true \ No newline at end of file diff --git a/pom.xml b/pom.xml index 49fa8290..e6c97436 100644 --- a/pom.xml +++ b/pom.xml @@ -15,6 +15,7 @@ Exporter for metrics, logs, and tracing from Oracle database 11 + 21.7.0.0 @@ -28,9 +29,28 @@ com.oracle.database.jdbc - ojdbc11-production - 21.3.0.0 - pom + ojdbc11 + ${oracle.jdbc.version} + + + com.oracle.database.jdbc + ucp + ${oracle.jdbc.version} + + + com.oracle.database.security + oraclepki + ${oracle.jdbc.version} + + + com.oracle.database.security + osdt_core + ${oracle.jdbc.version} + + + com.oracle.database.security + osdt_cert + ${oracle.jdbc.version} io.opentelemetry diff --git a/src/main/java/oracle/observability/ObservabilityExporter.java b/src/main/java/oracle/observability/ObservabilityExporter.java index f0209c25..4bf200f9 100644 --- a/src/main/java/oracle/observability/ObservabilityExporter.java +++ b/src/main/java/oracle/observability/ObservabilityExporter.java @@ -6,21 +6,32 @@ import com.oracle.bmc.secrets.model.Base64SecretBundleContentDetails; import com.oracle.bmc.secrets.requests.GetSecretBundleRequest; import com.oracle.bmc.secrets.responses.GetSecretBundleResponse; +import oracle.observability.metrics.MetricsExporter; import oracle.ucp.jdbc.PoolDataSource; import oracle.ucp.jdbc.PoolDataSourceFactory; import org.apache.commons.codec.binary.Base64; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.io.File; import java.io.IOException; import java.sql.SQLException; +import java.util.HashMap; +import java.util.Map; public class ObservabilityExporter { + private static final Logger LOGGER = LoggerFactory.getLogger(ObservabilityExporter.class); public String DEFAULT_METRICS = System.getenv("DEFAULT_METRICS"); // "default-metrics.toml" + public File DEFAULT_METRICS_FILE; public String CUSTOM_METRICS = System.getenv("CUSTOM_METRICS"); // public String QUERY_TIMEOUT = System.getenv("QUERY_TIMEOUT"); // "5" public String DATABASE_MAXIDLECONNS = System.getenv("DATABASE_MAXIDLECONNS"); // "0" public String DATABASE_MAXOPENCONNS = System.getenv("DATABASE_MAXOPENCONNS"); // "10" - public String DATA_SOURCE_NAME = System.getenv("DATA_SOURCE_NAME"); //eg %USER%/$(dbpassword)@%PDB_NAME%_tp + public static String DATA_SOURCE_NAME = System.getenv("DATA_SOURCE_NAME"); //eg %USER%/$(dbpassword)@%PDB_NAME%_tp + public static String DATA_SOURCE_USER = System.getenv("DATA_SOURCE_USER"); //eg %USER%/$(dbpassword)@%PDB_NAME%_tp + public static String DATA_SOURCE_PASSWORD = System.getenv("DATA_SOURCE_PASSWORD"); //eg %USER%/$(dbpassword)@%PDB_NAME%_tp + public static String DATA_SOURCE_SERVICENAME = System.getenv("DATA_SOURCE_SERVICENAME"); //eg %USER%/$(dbpassword)@%PDB_NAME%_tp public String TNS_ADMIN = System.getenv("TNS_ADMIN"); //eg /msdataworkshop/creds public String OCI_REGION = System.getenv("OCI_REGION"); //eg us-ashburn-1 public String VAULT_SECRET_OCID = System.getenv("VAULT_SECRET_OCID"); //eg ocid.... @@ -29,27 +40,52 @@ public class ObservabilityExporter { public static final String CONTEXT = "context"; public static final String REQUEST = "request"; + static { + + if (DATA_SOURCE_USER != null && DATA_SOURCE_PASSWORD != null && DATA_SOURCE_SERVICENAME != null) { + DATA_SOURCE_NAME = DATA_SOURCE_USER + "/" + DATA_SOURCE_PASSWORD + "@" + DATA_SOURCE_SERVICENAME; + LOGGER.info("DATA_SOURCE_NAME = DATA_SOURCE_USER + \"/\" + DATA_SOURCE_PASSWORD + \"@\" + DATA_SOURCE_SERVICENAME"); + //eg %USER%/$(dbpassword)@%PDB_NAME%_tp + } + } PoolDataSource observabilityDB; + Map dataSourceNameToDataSourceMap = new HashMap<>(); + public PoolDataSource getPoolDataSource() throws SQLException { - if (observabilityDB != null) return observabilityDB; - observabilityDB = PoolDataSourceFactory.getPoolDataSource(); - observabilityDB.setConnectionFactoryClassName("oracle.jdbc.pool.OracleDataSource"); - String user = DATA_SOURCE_NAME.substring(0, DATA_SOURCE_NAME.indexOf("/")); - String pw = DATA_SOURCE_NAME.substring(DATA_SOURCE_NAME.indexOf("/") + 1, DATA_SOURCE_NAME.indexOf("@")); - String serviceName = DATA_SOURCE_NAME.substring(DATA_SOURCE_NAME.indexOf("@") + 1); + return getPoolDataSource(DATA_SOURCE_NAME); + } + public PoolDataSource getPoolDataSource(String dataSourceName) throws SQLException { + if (dataSourceName.equals(DATA_SOURCE_NAME)) { + if (observabilityDB != null) return observabilityDB; + return observabilityDB = getDataSource(DATA_SOURCE_NAME); + } else { + if(dataSourceNameToDataSourceMap.containsKey(dataSourceName) && dataSourceNameToDataSourceMap.get(dataSourceName) != null) + return dataSourceNameToDataSourceMap.get(dataSourceName); + PoolDataSource poolDataSource = getDataSource(dataSourceName); + dataSourceNameToDataSourceMap.put(dataSourceName, poolDataSource); + return poolDataSource; + } + } + + private PoolDataSource getDataSource(String dataSourceName) throws SQLException { + PoolDataSource poolDataSource = PoolDataSourceFactory.getPoolDataSource(); + poolDataSource.setConnectionFactoryClassName("oracle.jdbc.pool.OracleDataSource"); + String user = dataSourceName.substring(0, dataSourceName.indexOf("/")); + String pw = dataSourceName.substring(dataSourceName.indexOf("/") + 1, dataSourceName.indexOf("@")); + String serviceName = dataSourceName.substring(dataSourceName.indexOf("@") + 1); String url = "jdbc:oracle:thin:@" + serviceName + "?TNS_ADMIN=" + TNS_ADMIN; - observabilityDB.setURL(url); - observabilityDB.setUser(user); - if (VAULT_SECRET_OCID == null || VAULT_SECRET_OCID.trim().equals("")) { - observabilityDB.setPassword(pw); + poolDataSource.setURL(url); + poolDataSource.setUser(user); + if (VAULT_SECRET_OCID == null || VAULT_SECRET_OCID.trim().equals("") || !dataSourceName.equals(DATA_SOURCE_NAME)) { + poolDataSource.setPassword(pw); } else { try { - observabilityDB.setPassword(getPasswordFromVault()); + poolDataSource.setPassword(getPasswordFromVault()); } catch (IOException e) { throw new SQLException(e); } } - return observabilityDB; + return poolDataSource; } diff --git a/src/main/java/oracle/observability/metrics/MetricsExporter.java b/src/main/java/oracle/observability/metrics/MetricsExporter.java index 5eb87e16..967273b2 100644 --- a/src/main/java/oracle/observability/metrics/MetricsExporter.java +++ b/src/main/java/oracle/observability/metrics/MetricsExporter.java @@ -7,6 +7,7 @@ import io.prometheus.client.Gauge; import oracle.observability.ObservabilityExporter; import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import org.slf4j.Logger; @@ -31,30 +32,38 @@ public class MetricsExporter extends ObservabilityExporter { public static final String LABELS = "labels"; public static final String IGNOREZERORESULT = "ignorezeroresult"; public static final String FALSE = "false"; - public String LISTEN_ADDRESS = System.getenv("LISTEN_ADDRESS"); // ":9161" - public String TELEMETRY_PATH = System.getenv("TELEMETRY_PATH"); // "/metrics" - //Interval between each scrape. Default is to scrape on collect requests. scrape.interval - public String SCRAPE_INTERVAL = System.getenv("scrape.interval"); // "0s" public static final String ORACLEDB_METRIC_PREFIX = "oracledb_"; Map gaugeMap = new HashMap<>(); + Map dnsToCollectorRegistryMap = new HashMap<>(); + + /** * The endpoint that prometheus will scrape * @return Prometheus metric - * @throws Exception */ @GetMapping(value = "/metrics", produces = "text/plain") public String metrics() throws Exception { - processMetrics(); - return getMetricsString(); + processMetrics(DATA_SOURCE_NAME, CollectorRegistry.defaultRegistry); + return getMetricsString(CollectorRegistry.defaultRegistry); + } + @GetMapping(value = "/scrape", produces = "text/plain") + public String scrape(@RequestParam("target") String target) throws Exception { + CollectorRegistry collectorRegistry = dnsToCollectorRegistryMap.get(target); + if (collectorRegistry == null) { + collectorRegistry = new CollectorRegistry(); + dnsToCollectorRegistryMap.put(target, collectorRegistry); + } + processMetrics(target, dnsToCollectorRegistryMap.get(target)); + return getMetricsString(collectorRegistry); } @PostConstruct public void init() throws Exception { - processMetrics(); + processMetrics(DATA_SOURCE_NAME, CollectorRegistry.defaultRegistry); } - private void processMetrics() throws IOException, SQLException { + private void processMetrics(String datasourceName, CollectorRegistry registry) throws IOException, SQLException { File tomlfile = new File(DEFAULT_METRICS); TomlMapper mapper = new TomlMapper(); JsonNode jsonNode = mapper.readerFor(MetricsExporterConfigEntry.class).readTree(new FileInputStream(tomlfile)); @@ -65,15 +74,15 @@ private void processMetrics() throws IOException, SQLException { } Iterator metrics = metric.iterator(); int isConnectionSuccessful = 0; - try(Connection connection = getPoolDataSource().getConnection()) { + try(Connection connection = getPoolDataSource(datasourceName).getConnection()) { isConnectionSuccessful = 1; while (metrics.hasNext()) { - processMetric(connection, metrics); + processMetric(registry, connection, metrics); } } finally { Gauge gauge = gaugeMap.get(ORACLEDB_METRIC_PREFIX + UP); if (gauge == null) { - Gauge upgauge = Gauge.build().name(ORACLEDB_METRIC_PREFIX + UP).help("Whether the Oracle database server is up.").register(); + Gauge upgauge = Gauge.build().name(ORACLEDB_METRIC_PREFIX + UP).help("Whether the Oracle database server is up.").register(registry); upgauge.set(isConnectionSuccessful); gaugeMap.put(ORACLEDB_METRIC_PREFIX + UP, upgauge); } else gauge.set(isConnectionSuccessful); @@ -91,7 +100,7 @@ private void processMetrics() throws IOException, SQLException { * Request string * IgnoreZeroResult bool */ - private void processMetric(Connection connection, Iterator metric) { + private void processMetric(CollectorRegistry registry, Connection connection, Iterator metric) { JsonNode next = metric.next(); String context = next.get(CONTEXT).asText(); // eg context = "teq" String metricsType = next.get(METRICSTYPE) == null ? "" :next.get(METRICSTYPE).asText(); @@ -120,7 +129,7 @@ private void processMetric(Connection connection, Iterator metric) { try { resultSet = connection.prepareStatement(request).executeQuery(); while (resultSet.next()) { - translateQueryToPrometheusMetric(context, metricsDescMap, labelNames, resultSet); + translateQueryToPrometheusMetric(registry, context, metricsDescMap, labelNames, resultSet); } } catch(SQLException e) { //this can be due to table not existing etc. LOGGER.warn("MetricsExporter.processMetric during:" + request + " exception:" + e); @@ -128,26 +137,19 @@ private void processMetric(Connection connection, Iterator metric) { } } - private void translateQueryToPrometheusMetric(String context, Map metricsDescMap, + private void translateQueryToPrometheusMetric(CollectorRegistry registry, String context, Map metricsDescMap, String[] labelNames, ResultSet resultSet) throws SQLException { String[] labelValues = new String[labelNames.length]; Map sqlQueryResults = - extractGaugesAndLabelValues(context, metricsDescMap, labelNames, resultSet, labelValues, resultSet.getMetaData().getColumnCount()); + extractGaugesAndLabelValues(registry, context, metricsDescMap, labelNames, resultSet, labelValues, resultSet.getMetaData().getColumnCount()); setLabelValues(context, labelNames, labelValues, sqlQueryResults.entrySet().iterator()); } /** * Creates Gauges and gets label values - * @param context - * @param metricsDescMap - * @param labelNames - * @param resultSet - * @param labelValues - * @param columnCount - * @throws SQLException */ - private Map extractGaugesAndLabelValues( + private Map extractGaugesAndLabelValues(CollectorRegistry registry, String context, Map metricsDescMap, String[] labelNames, ResultSet resultSet, String[] labelValues, int columnCount) throws SQLException { Map sqlQueryResults = new HashMap<>(); @@ -166,8 +168,8 @@ private Map extractGaugesAndLabelValues( if (gauge == null) { if(metricsDescMap.containsKey(columnName)) { if (labelNames.length > 0) { - gauge = Gauge.build().name(gaugeName.toLowerCase()).help(metricsDescMap.get(columnName)).labelNames(labelNames).register(); - } else gauge = Gauge.build().name(gaugeName.toLowerCase()).help(metricsDescMap.get(columnName)).register(); + gauge = Gauge.build().name(gaugeName.toLowerCase()).help(metricsDescMap.get(columnName)).labelNames(labelNames).register(registry); + } else gauge = Gauge.build().name(gaugeName.toLowerCase()).help(metricsDescMap.get(columnName)).register(registry); gaugeMap.put(gaugeName, gauge); } } @@ -198,8 +200,7 @@ private void setLabelValues(String context, String[] labelNames, String[] labelV } } - public static String getMetricsString() { - CollectorRegistry collectorRegistry = CollectorRegistry.defaultRegistry; + public static String getMetricsString(CollectorRegistry collectorRegistry) { Enumeration mfs = collectorRegistry.filteredMetricFamilySamples(new HashSet<>()); return compose(mfs); }