Skip to content

DATA_SOURCE_USER/PW/SERVICENAME support and multi-database support #18

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
paulparkinson merged 12 commits into from
Apr 27, 2023
Merged

DATA_SOURCE_USER/PW/SERVICENAME support and multi-database support #18

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
920705b
add list of default metrics in readme
paulparkinson Mar 23, 2022
76303d0
Merge remote-tracking branch 'upstream/main'
paulparkinson May 31, 2022
56f42d9
Merge remote-tracking branch 'upstream/main'
paulparkinson Jun 19, 2022
56903fe
readme updates
paulparkinson Jun 19, 2022
24d22c6
various updates
paulparkinson Jun 30, 2022
574e61a
README.md updates
paulparkinson Jul 3, 2022
dafa355
README.md updates
paulparkinson Jul 3, 2022
0b6b4fc
Merge remote-tracking branch 'upstream/main'
paulparkinson Jul 3, 2022
72a2c4a
basic example
paulparkinson Aug 3, 2022
0e1030d
Merge remote-tracking branch 'upstream/main'
paulparkinson Aug 3, 2022
9e1649a
adds... 1. ability to specify DATA_SOURCE_USER, DATA_SOURCE_PASSWORD,…
paulparkinson Apr 27, 2023
1036918
Merge remote-tracking branch 'upstream/main'
paulparkinson Apr 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,3 +15,4 @@ export IMAGE=${DOCKER_REGISTRY}/${IMAGE_NAME}:${IMAGE_VERSION}

mvn clean package -DskipTests
docker build -t=$IMAGE .
docker push $IMAGE
6 changes: 6 additions & 0 deletions examples/metrics/default-metrics.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
[[metric]]
context = "sessions"
labels = ["inst_id", "status", "type"]
metricsdesc = { value = "Gauge metric with count of sessions by status and type." }
request = "select inst_id, status, type, count(*) as value from gv$session group by status, type, inst_id"
ignorezeroresult = true
26 changes: 23 additions & 3 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
<description>Exporter for metrics, logs, and tracing from Oracle database</description>
<properties>
<java.version>11</java.version>
<oracle.jdbc.version>21.7.0.0</oracle.jdbc.version>
</properties>
<dependencies>
<dependency>
Expand All @@ -28,9 +29,28 @@
</dependency>
<dependency>
<groupId>com.oracle.database.jdbc</groupId>
<artifactId>ojdbc11-production</artifactId>
<version>21.3.0.0</version>
<type>pom</type>
<artifactId>ojdbc11</artifactId>
<version>${oracle.jdbc.version}</version>
</dependency>
<dependency>
<groupId>com.oracle.database.jdbc</groupId>
<artifactId>ucp</artifactId>
<version>${oracle.jdbc.version}</version>
</dependency>
<dependency>
<groupId>com.oracle.database.security</groupId>
<artifactId>oraclepki</artifactId>
<version>${oracle.jdbc.version}</version>
</dependency>
<dependency>
<groupId>com.oracle.database.security</groupId>
<artifactId>osdt_core</artifactId>
<version>${oracle.jdbc.version}</version>
</dependency>
<dependency>
<groupId>com.oracle.database.security</groupId>
<artifactId>osdt_cert</artifactId>
<version>${oracle.jdbc.version}</version>
</dependency>
<dependency>
<groupId>io.opentelemetry</groupId>
Expand Down
62 changes: 49 additions & 13 deletions src/main/java/oracle/observability/ObservabilityExporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,21 +6,32 @@
import com.oracle.bmc.secrets.model.Base64SecretBundleContentDetails;
import com.oracle.bmc.secrets.requests.GetSecretBundleRequest;
import com.oracle.bmc.secrets.responses.GetSecretBundleResponse;
import oracle.observability.metrics.MetricsExporter;
import oracle.ucp.jdbc.PoolDataSource;
import oracle.ucp.jdbc.PoolDataSourceFactory;
import org.apache.commons.codec.binary.Base64;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.IOException;
import java.sql.SQLException;
import java.util.HashMap;
import java.util.Map;

public class ObservabilityExporter {

private static final Logger LOGGER = LoggerFactory.getLogger(ObservabilityExporter.class);
public String DEFAULT_METRICS = System.getenv("DEFAULT_METRICS"); // "default-metrics.toml"
public File DEFAULT_METRICS_FILE;
public String CUSTOM_METRICS = System.getenv("CUSTOM_METRICS"); //
public String QUERY_TIMEOUT = System.getenv("QUERY_TIMEOUT"); // "5"
public String DATABASE_MAXIDLECONNS = System.getenv("DATABASE_MAXIDLECONNS"); // "0"
public String DATABASE_MAXOPENCONNS = System.getenv("DATABASE_MAXOPENCONNS"); // "10"
public String DATA_SOURCE_NAME = System.getenv("DATA_SOURCE_NAME"); //eg %USER%/$(dbpassword)@%PDB_NAME%_tp
public static String DATA_SOURCE_NAME = System.getenv("DATA_SOURCE_NAME"); //eg %USER%/$(dbpassword)@%PDB_NAME%_tp
public static String DATA_SOURCE_USER = System.getenv("DATA_SOURCE_USER"); //eg %USER%/$(dbpassword)@%PDB_NAME%_tp
public static String DATA_SOURCE_PASSWORD = System.getenv("DATA_SOURCE_PASSWORD"); //eg %USER%/$(dbpassword)@%PDB_NAME%_tp
public static String DATA_SOURCE_SERVICENAME = System.getenv("DATA_SOURCE_SERVICENAME"); //eg %USER%/$(dbpassword)@%PDB_NAME%_tp
public String TNS_ADMIN = System.getenv("TNS_ADMIN"); //eg /msdataworkshop/creds
public String OCI_REGION = System.getenv("OCI_REGION"); //eg us-ashburn-1
public String VAULT_SECRET_OCID = System.getenv("VAULT_SECRET_OCID"); //eg ocid....
Expand All @@ -29,27 +40,52 @@ public class ObservabilityExporter {
public static final String CONTEXT = "context";
public static final String REQUEST = "request";

static {

if (DATA_SOURCE_USER != null && DATA_SOURCE_PASSWORD != null && DATA_SOURCE_SERVICENAME != null) {
DATA_SOURCE_NAME = DATA_SOURCE_USER + "/" + DATA_SOURCE_PASSWORD + "@" + DATA_SOURCE_SERVICENAME;
LOGGER.info("DATA_SOURCE_NAME = DATA_SOURCE_USER + \"/\" + DATA_SOURCE_PASSWORD + \"@\" + DATA_SOURCE_SERVICENAME");
//eg %USER%/$(dbpassword)@%PDB_NAME%_tp
}
}
PoolDataSource observabilityDB;
Map<String, PoolDataSource> dataSourceNameToDataSourceMap = new HashMap<>();

public PoolDataSource getPoolDataSource() throws SQLException {
if (observabilityDB != null) return observabilityDB;
observabilityDB = PoolDataSourceFactory.getPoolDataSource();
observabilityDB.setConnectionFactoryClassName("oracle.jdbc.pool.OracleDataSource");
String user = DATA_SOURCE_NAME.substring(0, DATA_SOURCE_NAME.indexOf("/"));
String pw = DATA_SOURCE_NAME.substring(DATA_SOURCE_NAME.indexOf("/") + 1, DATA_SOURCE_NAME.indexOf("@"));
String serviceName = DATA_SOURCE_NAME.substring(DATA_SOURCE_NAME.indexOf("@") + 1);
return getPoolDataSource(DATA_SOURCE_NAME);
}
public PoolDataSource getPoolDataSource(String dataSourceName) throws SQLException {
if (dataSourceName.equals(DATA_SOURCE_NAME)) {
if (observabilityDB != null) return observabilityDB;
return observabilityDB = getDataSource(DATA_SOURCE_NAME);
} else {
if(dataSourceNameToDataSourceMap.containsKey(dataSourceName) && dataSourceNameToDataSourceMap.get(dataSourceName) != null)
return dataSourceNameToDataSourceMap.get(dataSourceName);
PoolDataSource poolDataSource = getDataSource(dataSourceName);
dataSourceNameToDataSourceMap.put(dataSourceName, poolDataSource);
return poolDataSource;
}
}

private PoolDataSource getDataSource(String dataSourceName) throws SQLException {
PoolDataSource poolDataSource = PoolDataSourceFactory.getPoolDataSource();
poolDataSource.setConnectionFactoryClassName("oracle.jdbc.pool.OracleDataSource");
String user = dataSourceName.substring(0, dataSourceName.indexOf("/"));
String pw = dataSourceName.substring(dataSourceName.indexOf("/") + 1, dataSourceName.indexOf("@"));
String serviceName = dataSourceName.substring(dataSourceName.indexOf("@") + 1);
String url = "jdbc:oracle:thin:@" + serviceName + "?TNS_ADMIN=" + TNS_ADMIN;
observabilityDB.setURL(url);
observabilityDB.setUser(user);
if (VAULT_SECRET_OCID == null || VAULT_SECRET_OCID.trim().equals("")) {
observabilityDB.setPassword(pw);
poolDataSource.setURL(url);
poolDataSource.setUser(user);
if (VAULT_SECRET_OCID == null || VAULT_SECRET_OCID.trim().equals("") || !dataSourceName.equals(DATA_SOURCE_NAME)) {
poolDataSource.setPassword(pw);
} else {
try {
observabilityDB.setPassword(getPasswordFromVault());
poolDataSource.setPassword(getPasswordFromVault());
} catch (IOException e) {
throw new SQLException(e);
}
}
return observabilityDB;
return poolDataSource;
}


Expand Down
57 changes: 29 additions & 28 deletions src/main/java/oracle/observability/metrics/MetricsExporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import io.prometheus.client.Gauge;
import oracle.observability.ObservabilityExporter;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;

import org.slf4j.Logger;
Expand All @@ -31,30 +32,38 @@ public class MetricsExporter extends ObservabilityExporter {
public static final String LABELS = "labels";
public static final String IGNOREZERORESULT = "ignorezeroresult";
public static final String FALSE = "false";
public String LISTEN_ADDRESS = System.getenv("LISTEN_ADDRESS"); // ":9161"
public String TELEMETRY_PATH = System.getenv("TELEMETRY_PATH"); // "/metrics"
//Interval between each scrape. Default is to scrape on collect requests. scrape.interval
public String SCRAPE_INTERVAL = System.getenv("scrape.interval"); // "0s"
public static final String ORACLEDB_METRIC_PREFIX = "oracledb_";
Map<String, Gauge> gaugeMap = new HashMap<>();
Map<String, CollectorRegistry> dnsToCollectorRegistryMap = new HashMap<>();



/**
* The endpoint that prometheus will scrape
* @return Prometheus metric
* @throws Exception
*/
@GetMapping(value = "/metrics", produces = "text/plain")
public String metrics() throws Exception {
processMetrics();
return getMetricsString();
processMetrics(DATA_SOURCE_NAME, CollectorRegistry.defaultRegistry);
return getMetricsString(CollectorRegistry.defaultRegistry);
}
@GetMapping(value = "/scrape", produces = "text/plain")
public String scrape(@RequestParam("target") String target) throws Exception {
CollectorRegistry collectorRegistry = dnsToCollectorRegistryMap.get(target);
if (collectorRegistry == null) {
collectorRegistry = new CollectorRegistry();
dnsToCollectorRegistryMap.put(target, collectorRegistry);
}
processMetrics(target, dnsToCollectorRegistryMap.get(target));
return getMetricsString(collectorRegistry);
}

@PostConstruct
public void init() throws Exception {
processMetrics();
processMetrics(DATA_SOURCE_NAME, CollectorRegistry.defaultRegistry);
}

private void processMetrics() throws IOException, SQLException {
private void processMetrics(String datasourceName, CollectorRegistry registry) throws IOException, SQLException {
File tomlfile = new File(DEFAULT_METRICS);
TomlMapper mapper = new TomlMapper();
JsonNode jsonNode = mapper.readerFor(MetricsExporterConfigEntry.class).readTree(new FileInputStream(tomlfile));
Expand All @@ -65,15 +74,15 @@ private void processMetrics() throws IOException, SQLException {
}
Iterator<JsonNode> metrics = metric.iterator();
int isConnectionSuccessful = 0;
try(Connection connection = getPoolDataSource().getConnection()) {
try(Connection connection = getPoolDataSource(datasourceName).getConnection()) {
isConnectionSuccessful = 1;
while (metrics.hasNext()) {
processMetric(connection, metrics);
processMetric(registry, connection, metrics);
}
} finally {
Gauge gauge = gaugeMap.get(ORACLEDB_METRIC_PREFIX + UP);
if (gauge == null) {
Gauge upgauge = Gauge.build().name(ORACLEDB_METRIC_PREFIX + UP).help("Whether the Oracle database server is up.").register();
Gauge upgauge = Gauge.build().name(ORACLEDB_METRIC_PREFIX + UP).help("Whether the Oracle database server is up.").register(registry);
upgauge.set(isConnectionSuccessful);
gaugeMap.put(ORACLEDB_METRIC_PREFIX + UP, upgauge);
} else gauge.set(isConnectionSuccessful);
Expand All @@ -91,7 +100,7 @@ private void processMetrics() throws IOException, SQLException {
* Request string
* IgnoreZeroResult bool
*/
private void processMetric(Connection connection, Iterator<JsonNode> metric) {
private void processMetric(CollectorRegistry registry, Connection connection, Iterator<JsonNode> metric) {
JsonNode next = metric.next();
String context = next.get(CONTEXT).asText(); // eg context = "teq"
String metricsType = next.get(METRICSTYPE) == null ? "" :next.get(METRICSTYPE).asText();
Expand Down Expand Up @@ -120,34 +129,27 @@ private void processMetric(Connection connection, Iterator<JsonNode> metric) {
try {
resultSet = connection.prepareStatement(request).executeQuery();
while (resultSet.next()) {
translateQueryToPrometheusMetric(context, metricsDescMap, labelNames, resultSet);
translateQueryToPrometheusMetric(registry, context, metricsDescMap, labelNames, resultSet);
}
} catch(SQLException e) { //this can be due to table not existing etc.
LOGGER.warn("MetricsExporter.processMetric during:" + request + " exception:" + e);
return;
}
}

private void translateQueryToPrometheusMetric(String context, Map<String, String> metricsDescMap,
private void translateQueryToPrometheusMetric(CollectorRegistry registry, String context, Map<String, String> metricsDescMap,
String[] labelNames,
ResultSet resultSet) throws SQLException {
String[] labelValues = new String[labelNames.length];
Map<String, Long> sqlQueryResults =
extractGaugesAndLabelValues(context, metricsDescMap, labelNames, resultSet, labelValues, resultSet.getMetaData().getColumnCount());
extractGaugesAndLabelValues(registry, context, metricsDescMap, labelNames, resultSet, labelValues, resultSet.getMetaData().getColumnCount());
setLabelValues(context, labelNames, labelValues, sqlQueryResults.entrySet().iterator());
}

/**
* Creates Gauges and gets label values
* @param context
* @param metricsDescMap
* @param labelNames
* @param resultSet
* @param labelValues
* @param columnCount
* @throws SQLException
*/
private Map<String, Long> extractGaugesAndLabelValues(
private Map<String, Long> extractGaugesAndLabelValues(CollectorRegistry registry,
String context, Map<String, String> metricsDescMap, String[] labelNames, ResultSet resultSet,
String[] labelValues, int columnCount) throws SQLException {
Map<String, Long> sqlQueryResults = new HashMap<>();
Expand All @@ -166,8 +168,8 @@ private Map<String, Long> extractGaugesAndLabelValues(
if (gauge == null) {
if(metricsDescMap.containsKey(columnName)) {
if (labelNames.length > 0) {
gauge = Gauge.build().name(gaugeName.toLowerCase()).help(metricsDescMap.get(columnName)).labelNames(labelNames).register();
} else gauge = Gauge.build().name(gaugeName.toLowerCase()).help(metricsDescMap.get(columnName)).register();
gauge = Gauge.build().name(gaugeName.toLowerCase()).help(metricsDescMap.get(columnName)).labelNames(labelNames).register(registry);
} else gauge = Gauge.build().name(gaugeName.toLowerCase()).help(metricsDescMap.get(columnName)).register(registry);
gaugeMap.put(gaugeName, gauge);
}
}
Expand Down Expand Up @@ -198,8 +200,7 @@ private void setLabelValues(String context, String[] labelNames, String[] labelV
}
}

public static String getMetricsString() {
CollectorRegistry collectorRegistry = CollectorRegistry.defaultRegistry;
public static String getMetricsString(CollectorRegistry collectorRegistry) {
Enumeration<Collector.MetricFamilySamples> mfs = collectorRegistry.filteredMetricFamilySamples(new HashSet<>());
return compose(mfs);
}
Expand Down