added a jdbc based implementation of metrics - both producing and con…

…suming
apiman · Apr 5, 2016 · 3438771 · 3438771
1 parent 5f72115
commit 3438771
Show file tree

Hide file tree

Showing 32 changed files with 2,321 additions and 37 deletions.
diff --git a/distro/data/src/main/resources/ddls/apiman-gateway_h2.ddl b/distro/data/src/main/resources/ddls/apiman-gateway_h2.ddl
@@ -8,3 +8,14 @@ ALTER TABLE clients ADD CONSTRAINT UK_clients_1 UNIQUE (org_id, id, version);
 CREATE INDEX IDX_clients_1 ON clients(org_id, id, version);
 
 CREATE TABLE dataversion (version BIGINT NOT NULL);
+
+CREATE TABLE requests (
+	rstart BIGINT NOT NULL, rend BIGINT NOT NULL, duration BIGINT NOT NULL, 
+	month BIGINT NOT NULL, week BIGINT NOT NULL, day BIGINT NOT NULL, hour BIGINT NOT NULL, minute BIGINT NOT NULL,
+	api_org_id VARCHAR(255) NOT NULL, api_id VARCHAR(255) NOT NULL, api_version VARCHAR(255) NOT NULL, 
+	client_org_id VARCHAR(255), client_id VARCHAR(255), client_version VARCHAR(255), plan VARCHAR(255),
+	user_id VARCHAR(255), resp_type VARCHAR(255), bytes_up BIGINT NOT NULL, bytes_down BIGINT NOT NULL
+);
+CREATE INDEX IDX_requests_1 ON requests(api_org_id, api_id, api_version);
+CREATE INDEX IDX_requests_2 ON requests(client_org_id, client_id, client_version);
+CREATE INDEX IDX_requests_3 ON requests(resp_type);
diff --git a/distro/data/src/main/resources/ddls/apiman-gateway_mysql5.ddl b/distro/data/src/main/resources/ddls/apiman-gateway_mysql5.ddl
@@ -2,9 +2,20 @@
 CREATE TABLE apis (org_id VARCHAR(255) NOT NULL, id VARCHAR(255) NOT NULL, version VARCHAR(255) NOT NULL, bean LONGTEXT NOT NULL);
 ALTER TABLE apis ADD PRIMARY KEY (org_id, id, version);
 
-CREATE TABLE clients (org_id VARCHAR(255) NOT NULL, id VARCHAR(255) NOT NULL, version VARCHAR(255) NOT NULL, bean LONGTEXT NOT NULL);
+CREATE TABLE clients (api_key VARCHAR(255) NOT NULL, org_id VARCHAR(255) NOT NULL, id VARCHAR(255) NOT NULL, version VARCHAR(255) NOT NULL, bean LONGTEXT NOT NULL);
 ALTER TABLE clients ADD PRIMARY KEY (api_key);
 ALTER TABLE clients ADD CONSTRAINT UK_clients_1 UNIQUE (org_id, id, version);
 CREATE INDEX IDX_clients_1 ON clients(org_id, id, version);
 
 CREATE TABLE dataversion (version BIGINT NOT NULL);
+
+CREATE TABLE requests (
+	rstart BIGINT NOT NULL, rend BIGINT NOT NULL, duration BIGINT NOT NULL, 
+	month BIGINT NOT NULL, week BIGINT NOT NULL, day BIGINT NOT NULL, hour BIGINT NOT NULL, minute BIGINT NOT NULL,
+	api_org_id VARCHAR(255) NOT NULL, api_id VARCHAR(255) NOT NULL, api_version VARCHAR(255) NOT NULL, 
+	client_org_id VARCHAR(255), client_id VARCHAR(255), client_version VARCHAR(255), plan VARCHAR(255),
+	user_id VARCHAR(255), resp_type VARCHAR(255), bytes_up BIGINT NOT NULL, bytes_down BIGINT NOT NULL
+);
+CREATE INDEX IDX_requests_1 ON requests(api_org_id, api_id, api_version);
+CREATE INDEX IDX_requests_2 ON requests(client_org_id, client_id, client_version);
+CREATE INDEX IDX_requests_3 ON requests(resp_type);
diff --git a/distro/data/src/main/resources/ddls/apiman-gateway_oracle12.ddl b/distro/data/src/main/resources/ddls/apiman-gateway_oracle12.ddl
@@ -2,9 +2,20 @@
 CREATE TABLE apis (org_id VARCHAR2(255) NOT NULL, id VARCHAR2(255) NOT NULL, version VARCHAR2(255) NOT NULL, bean CLOB NOT NULL);
 ALTER TABLE apis ADD PRIMARY KEY (org_id, id, version);
 
-CREATE TABLE clients (org_id VARCHAR2(255) NOT NULL, id VARCHAR2(255) NOT NULL, version VARCHAR2(255) NOT NULL, bean CLOB NOT NULL);
+CREATE TABLE clients (api_key VARCHAR2(255) NOT NULL, org_id VARCHAR2(255) NOT NULL, id VARCHAR2(255) NOT NULL, version VARCHAR2(255) NOT NULL, bean CLOB NOT NULL);
 ALTER TABLE clients ADD PRIMARY KEY (api_key);
 ALTER TABLE clients ADD CONSTRAINT UK_clients_1 UNIQUE (org_id, id, version);
 CREATE INDEX IDX_clients_1 ON clients(org_id, id, version);
 
 CREATE TABLE dataversion (version NUMBER(38, 0) NOT NULL);
+
+CREATE TABLE requests (
+	rstart BIGINT NOT NULL, rend BIGINT NOT NULL, duration BIGINT NOT NULL, 
+	month BIGINT NOT NULL, week BIGINT NOT NULL, day BIGINT NOT NULL, hour BIGINT NOT NULL, minute BIGINT NOT NULL,
+	api_org_id VARCHAR2(255) NOT NULL, api_id VARCHAR2(255) NOT NULL, api_version VARCHAR2(255) NOT NULL, 
+	client_org_id VARCHAR2(255), client_id VARCHAR2(255), client_version VARCHAR2(255), plan VARCHAR2(255),
+	user_id VARCHAR2(255), resp_type VARCHAR2(255), bytes_up BIGINT NOT NULL, bytes_down BIGINT NOT NULL
+);
+CREATE INDEX IDX_requests_1 ON requests(api_org_id, api_id, api_version);
+CREATE INDEX IDX_requests_2 ON requests(client_org_id, client_id, client_version);
+CREATE INDEX IDX_requests_3 ON requests(resp_type);
diff --git a/distro/data/src/main/resources/ddls/apiman-gateway_postgresql9.ddl b/distro/data/src/main/resources/ddls/apiman-gateway_postgresql9.ddl
@@ -2,9 +2,20 @@
 CREATE TABLE apis (org_id VARCHAR(255) NOT NULL, id VARCHAR(255) NOT NULL, version VARCHAR(255) NOT NULL, bean TEXT NOT NULL);
 ALTER TABLE apis ADD PRIMARY KEY (org_id, id, version);
 
-CREATE TABLE clients (org_id VARCHAR(255) NOT NULL, id VARCHAR(255) NOT NULL, version VARCHAR(255) NOT NULL, bean TEXT NOT NULL);
+CREATE TABLE clients (api_key VARCHAR(255) NOT NULL, org_id VARCHAR(255) NOT NULL, id VARCHAR(255) NOT NULL, version VARCHAR(255) NOT NULL, bean TEXT NOT NULL);
 ALTER TABLE clients ADD PRIMARY KEY (api_key);
 ALTER TABLE clients ADD CONSTRAINT UK_clients_1 UNIQUE (org_id, id, version);
 CREATE INDEX IDX_clients_1 ON clients(org_id, id, version);
 
 CREATE TABLE dataversion (version BIGINT NOT NULL);
+
+CREATE TABLE requests (
+	rstart BIGINT NOT NULL, rend BIGINT NOT NULL, duration BIGINT NOT NULL, 
+	month BIGINT NOT NULL, week BIGINT NOT NULL, day BIGINT NOT NULL, hour BIGINT NOT NULL, minute BIGINT NOT NULL,
+	api_org_id VARCHAR(255) NOT NULL, api_id VARCHAR(255) NOT NULL, api_version VARCHAR(255) NOT NULL, 
+	client_org_id VARCHAR(255), client_id VARCHAR(255), client_version VARCHAR(255), plan VARCHAR(255),
+	user_id VARCHAR(255), resp_type VARCHAR(255), bytes_up BIGINT NOT NULL, bytes_down BIGINT NOT NULL
+);
+CREATE INDEX IDX_requests_1 ON requests(api_org_id, api_id, api_version);
+CREATE INDEX IDX_requests_2 ON requests(client_org_id, client_id, client_version);
+CREATE INDEX IDX_requests_3 ON requests(resp_type);
diff --git a/distro/data/src/main/resources/sample-configs/apiman_gw-ds_mysql.xml b/distro/data/src/main/resources/sample-configs/apiman_gw-ds_mysql.xml
@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<datasources>
+  <datasource jndi-name="java:jboss/datasources/apiman-gateway" pool-name="apiman-gateway" enabled="true"
+    use-java-context="true">
+    <connection-url>jdbc:mysql://MYSQLSERVER:3306/apiman_gateway</connection-url>
+    <driver>mysql-connector-java-5.1.33-bin.jar_com.mysql.jdbc.Driver_5_1</driver>
+    <pool>
+      <max-pool-size>30</max-pool-size>
+    </pool>
+    <security>
+      <user-name>DBUSER</user-name>
+      <password>DBPASS</password>
+    </security>
+  </datasource>
+</datasources>
diff --git a/distro/data/src/main/resources/sample-configs/apiman_gw-ds_postgresql.xml b/distro/data/src/main/resources/sample-configs/apiman_gw-ds_postgresql.xml
@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<datasources>
+  <datasource jndi-name="java:jboss/datasources/apiman-gateway" pool-name="apiman-gateway" enabled="true" use-java-context="true">
+    <connection-url>jdbc:postgresql://POSTGRESQLSERVER:5432/apiman_gateway</connection-url>
+    <driver>postgresql-9.3-1102.jdbc41.jar</driver>
+    <pool>
+      <max-pool-size>30</max-pool-size>
+    </pool>
+    <security>
+      <user-name>DBUSER</user-name>
+      <password>DBPASS</password>
+    </security>
+  </datasource>
+</datasources>
diff --git a/gateway/engine/es/src/main/java/io/apiman/gateway/engine/es/ESMetrics.java b/gateway/engine/es/src/main/java/io/apiman/gateway/engine/es/ESMetrics.java
@@ -46,7 +46,7 @@ public ESMetrics(Map<String, String> config) {
         super(config);
 
         int queueSize = DEFAULT_QUEUE_SIZE;
-        String queueSizeConfig = config.get("es.queueSize"); //$NON-NLS-1$
+        String queueSizeConfig = config.get("queue.size"); //$NON-NLS-1$
         if (queueSizeConfig != null) {
             queueSize = new Integer(queueSizeConfig);
         }

diff --git a/gateway/engine/jdbc/pom.xml b/gateway/engine/jdbc/pom.xml
@@ -50,6 +50,53 @@
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-databind</artifactId>
     </dependency>
+
+    <!-- Test Only Libs -->
+    <dependency>
+      <groupId>${project.groupId}</groupId>
+      <artifactId>apiman-test-common</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>${project.groupId}</groupId>
+      <artifactId>apiman-distro-data</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>commons-io</groupId>
+      <artifactId>commons-io</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>commons-dbcp</groupId>
+      <artifactId>commons-dbcp</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>commons-pool</groupId>
+      <artifactId>commons-pool</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.h2database</groupId>
+      <artifactId>h2</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <artifactId>slf4j-log4j12</artifactId>
+      <groupId>org.slf4j</groupId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.jboss.logging</groupId>
+      <artifactId>jboss-logging</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>

diff --git a/gateway/engine/jdbc/src/main/java/io/apiman/gateway/engine/jdbc/AbstractJdbcComponent.java b/gateway/engine/jdbc/src/main/java/io/apiman/gateway/engine/jdbc/AbstractJdbcComponent.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2016 JBoss Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.apiman.gateway.engine.jdbc;
+
+import java.util.Map;
+
+import javax.naming.InitialContext;
+import javax.sql.DataSource;
+
+/**
+ * A base class for all JDBC component implementations.  Handles the configuration
+ * of the required datasource.
+ * 
+ * @author eric.wittmann@gmail.com
+ */
+public abstract class AbstractJdbcComponent {
+
+    protected DataSource ds;
+
+    /**
+     * Constructor.
+     * @param config map of configuration options
+     */
+    public AbstractJdbcComponent(Map<String, String> config) {
+        String dsJndiLocation = config.get("datasource.jndi-location"); //$NON-NLS-1$
+        if (dsJndiLocation == null) {
+            throw new RuntimeException("Missing datasource JNDI location from JdbcRegistry configuration."); //$NON-NLS-1$
+        }
+        ds = lookupDS(dsJndiLocation);
+    }
+
+    /**
+     * Lookup the datasource in JNDI.
+     * @param dsJndiLocation
+     */
+    private static DataSource lookupDS(String dsJndiLocation) {
+        DataSource ds;
+        try {
+            InitialContext ctx = new InitialContext();
+            ds = (DataSource) ctx.lookup(dsJndiLocation);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+
+        if (ds == null) {
+            throw new RuntimeException("Datasource not found: " + dsJndiLocation); //$NON-NLS-1$
+        }
+        return ds;
+    }
+
+}