Skip to content

Commit

Permalink
TEIID-4122 improving the metadata query performance and enhancing
Browse files Browse the repository at this point in the history 
compatiblity
  • Loading branch information
@shawkins
shawkins committed Apr 20, 2016
1 parent 831fcf7 commit c2b270d
Show file tree
Hide file tree
Showing 24 changed files with 657 additions and 612 deletions.
28 changes: 20 additions & 8 deletions runtime/src/main/java/org/teiid/deployers/PgCatalogMetadataStore.java
View file
Expand Up @@ -45,12 +45,16 @@
import org.teiid.metadata.Table;
import org.teiid.metadata.Table.Type;
import org.teiid.odbc.ODBCServerRemoteImpl;
import org.teiid.query.metadata.MaterializationMetadataRepository;
import org.teiid.query.metadata.TransformationMetadata;
import org.teiid.query.resolver.util.ResolverVisitor;
import org.teiid.transport.PgBackendProtocol;

public class PgCatalogMetadataStore extends MetadataFactory {
private static final long serialVersionUID = 2158418324376966987L;

public static final String TYPMOD = "(CASE WHEN (t1.DataType = 'bigdecimal' OR t1.DataType = 'biginteger') THEN (CASE WHEN (4+(cast(65536 as float)*t1.Precision)+t1.Scale) > 2147483647 THEN 2147483647 ELSE (4+(65536*t1.Precision)+t1.Scale) END) " + //$NON-NLS-1$
"WHEN (t1.DataType = 'string' OR t1.DataType = 'char') THEN (CASE WHEN (t1.Length <= 2147483643) THEN 4+ t1.Length ELSE 2147483647 END) ELSE -1 END)"; //$NON-NLS-1$

public PgCatalogMetadataStore(String modelName, Map<String, Datatype> dataTypes) {
super(modelName, 1, modelName, dataTypes, new Properties(), null);
Expand Down Expand Up @@ -172,8 +176,7 @@ private Table add_pg_attribute() {
"pt.oid as atttypid," + //$NON-NLS-1$
"pt.typlen as attlen, " + //$NON-NLS-1$
"convert(t1.Position, short) as attnum, " + //$NON-NLS-1$
"(CASE WHEN (t1.DataType = 'bigdecimal' OR t1.DataType = 'biginteger') THEN (CASE WHEN (4+(cast(65536 as float)*t1.Precision)+t1.Scale) > 2147483647 THEN 2147483647 ELSE (4+(65536*t1.Precision)+t1.Scale) END) " + //$NON-NLS-1$
"WHEN (t1.DataType = 'string' OR t1.DataType = 'char') THEN (CASE WHEN (t1.Length <= 2147483643) THEN 4+ t1.Length ELSE 2147483647 END) ELSE -1 END) as atttypmod, " + //$NON-NLS-1$
TYPMOD +" as atttypmod, " + //$NON-NLS-1$
"CASE WHEN (t1.NullType = 'No Nulls') THEN true ELSE false END as attnotnull, " + //$NON-NLS-1$
"false as attisdropped, " + //$NON-NLS-1$
"false as atthasdef " + //$NON-NLS-1$
Expand All @@ -185,8 +188,7 @@ private Table add_pg_attribute() {
"pt.oid as atttypid," + //$NON-NLS-1$
"pt.typlen as attlen, " + //$NON-NLS-1$
"convert(kc.Position, short) as attnum, " + //$NON-NLS-1$
"(CASE WHEN (t1.DataType = 'bigdecimal' OR t1.DataType = 'biginteger') THEN (CASE WHEN (4+(cast(65536 as float)*t1.Precision)+t1.Scale) > 2147483647 THEN 2147483647 ELSE (4+(65536*t1.Precision)+t1.Scale) END) " + //$NON-NLS-1$
"WHEN (t1.DataType = 'string' OR t1.DataType = 'char') THEN (CASE WHEN (t1.Length <= 2147483643) THEN 4+ t1.Length ELSE 2147483647 END) ELSE -1 END) as atttypmod, " + //$NON-NLS-1$
TYPMOD +" as atttypmod, " + //$NON-NLS-1$
"CASE WHEN (t1.NullType = 'No Nulls') THEN true ELSE false END as attnotnull, " + //$NON-NLS-1$
"false as attisdropped, " + //$NON-NLS-1$
"false as atthasdef " + //$NON-NLS-1$
Expand Down Expand Up @@ -226,7 +228,10 @@ private Table add_pg_class() {
addColumn("relhasrules", DataTypeManager.DefaultDataTypes.BOOLEAN, t); //$NON-NLS-1$

// True if we generate an OID for each row of the relation
addColumn("relhasoids", DataTypeManager.DefaultDataTypes.BOOLEAN, t); //$NON-NLS-1$
addColumn("relhasoids", DataTypeManager.DefaultDataTypes.BOOLEAN, t); //$NON-NLS-1$

//additional column not present in pg metadata - for column metadata query
addColumn("relnspname", DataTypeManager.DefaultDataTypes.STRING, t); //$NON-NLS-1$

addPrimaryKey("pk_pg_class", Arrays.asList("oid"), t); //$NON-NLS-1$ //$NON-NLS-2$

Expand All @@ -237,15 +242,17 @@ private Table add_pg_class() {
"convert(0, float) as reltuples, " + //$NON-NLS-1$
"0 as relpages, " + //$NON-NLS-1$
"false as relhasrules, " + //$NON-NLS-1$
"false as relhasoids " + //$NON-NLS-1$
"false as relhasoids, " + //$NON-NLS-1$
"t1.SchemaName as relnspname " + //$NON-NLS-1$
"FROM SYS.Tables t1 UNION ALL SELECT pg_catalog.getOid(t1.uid) as oid, t1.name as relname, " + //$NON-NLS-1$
"pg_catalog.getOid(uid) as relnamespace, " + //$NON-NLS-1$
"convert('i', char) as relkind," + //$NON-NLS-1$
"0 as relam, " + //$NON-NLS-1$
"convert(0, float) as reltuples, " + //$NON-NLS-1$
"0 as relpages, " + //$NON-NLS-1$
"false as relhasrules, " + //$NON-NLS-1$
"false as relhasoids " + //$NON-NLS-1$
"false as relhasoids, " + //$NON-NLS-1$
"t1.SchemaName as relnspname " + //$NON-NLS-1$
"FROM SYS.Keys t1 WHERE t1.type in ('Primary', 'Unique', 'Index')"; //$NON-NLS-1$
t.setSelectTransformation(transformation);
return t;
Expand Down Expand Up @@ -476,6 +483,7 @@ private Table add_pg_type() {
"' columns oid integer, typname string, typlen short, typtype char, typbasetype integer, typtypmod integer, typrelid integer, typelem integer) AS t"; //$NON-NLS-1$
t.setSelectTransformation(transformation);
t.setMaterialized(true);
t.setProperty(MaterializationMetadataRepository.ALLOW_MATVIEW_MANAGEMENT, "true"); //$NON-NLS-1$
return t;
}

Expand Down Expand Up @@ -527,10 +535,13 @@ private Table add_matpg_datatype() {
addColumn("name", DataTypeManager.DefaultDataTypes.STRING, t); //$NON-NLS-1$
addColumn("uid", DataTypeManager.DefaultDataTypes.STRING, t); //$NON-NLS-1$
addColumn("typlen", DataTypeManager.DefaultDataTypes.SHORT, t); //$NON-NLS-1$
addColumn("typtype", DataTypeManager.DefaultDataTypes.CHAR, t); //$NON-NLS-1$
addColumn("typbasetype", DataTypeManager.DefaultDataTypes.INTEGER, t); //$NON-NLS-1$
addColumn("typtypmod", DataTypeManager.DefaultDataTypes.INTEGER, t); //$NON-NLS-1$

addPrimaryKey("matpg_datatype_names", Arrays.asList("oid", "name"), t); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
addIndex("matpg_datatype_ids", true, Arrays.asList("typname", "oid"), t); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
String transformation = "select pt.oid as oid, pt.typname as typname, t.Name name, t.UID, pt.typlen from pg_catalog.pg_type pt JOIN (select (CASE "+//$NON-NLS-1$
String transformation = "select pt.oid as oid, pt.typname as typname, t.Name name, t.UID, pt.typlen, pt.typtype, pt.typbasetype, pt.typtypmod from pg_catalog.pg_type pt JOIN (select (CASE "+//$NON-NLS-1$
"WHEN (Name = 'clob' OR Name = 'blob') THEN 'lo' " +//$NON-NLS-1$
"WHEN (Name = 'byte' ) THEN 'short' " +//$NON-NLS-1$
"WHEN (Name = 'time' ) THEN 'datetime' " + //$NON-NLS-1$
Expand All @@ -540,6 +551,7 @@ private Table add_matpg_datatype() {
"ELSE Name END) as pg_name, Name, UID from SYS.DataTypes) as t ON t.pg_name = pt.typname"; //$NON-NLS-1$
t.setSelectTransformation(transformation);
t.setMaterialized(true);
t.setProperty(MaterializationMetadataRepository.ALLOW_MATVIEW_MANAGEMENT, "true"); //$NON-NLS-1$
return t;
}

Expand Down
21 changes: 18 additions & 3 deletions runtime/src/main/java/org/teiid/odbc/ODBCServerRemoteImpl.java
View file
Expand Up @@ -84,6 +84,12 @@ public class ODBCServerRemoteImpl implements ODBCServerRemote {
private static final String UNNAMED = ""; //$NON-NLS-1$
private static Pattern setPattern = Pattern.compile("set\\s+(\\w+)\\s+to\\s+((?:'[^']*')+)", Pattern.DOTALL|Pattern.CASE_INSENSITIVE);//$NON-NLS-1$

private static Pattern columnMetadataPattern = Pattern.compile("select n.nspname, c.relname, a.attname, a.atttypid, t.typname, a.attnum, a.attlen, a.atttypmod, a.attnotnull, " //$NON-NLS-1$
+ "c.relhasrules, c.relkind, c.oid, pg_get_expr\\(d.adbin, d.adrelid\\), case t.typtype when 'd' then t.typbasetype else 0 end, t.typtypmod, c.relhasoids " //$NON-NLS-1$
+ "from \\(\\(\\(pg_catalog.pg_class c inner join pg_catalog.pg_namespace n on n.oid = c.relnamespace and c.oid = (\\d+)\\) inner join pg_catalog.pg_attribute a " //$NON-NLS-1$
+ "on \\(not a.attisdropped\\) and a.attnum > 0 and a.attrelid = c.oid\\) inner join pg_catalog.pg_type t on t.oid = a.atttypid\\) left outer join pg_attrdef d " //$NON-NLS-1$
+ "on a.atthasdef and d.adrelid = a.attrelid and d.adnum = a.attnum order by n.nspname, c.relname, attnum"); //$NON-NLS-1$

private static Pattern pkPattern = Pattern.compile("select ta.attname, ia.attnum, ic.relname, n.nspname, tc.relname " +//$NON-NLS-1$
"from pg_catalog.pg_attribute ta, pg_catalog.pg_attribute ia, pg_catalog.pg_class tc, pg_catalog.pg_index i, " +//$NON-NLS-1$
"pg_catalog.pg_namespace n, pg_catalog.pg_class ic where tc.relname = (E?(?:'[^']*')+) AND n.nspname = (E?(?:'[^']*')+).*", Pattern.DOTALL|Pattern.CASE_INSENSITIVE);//$NON-NLS-1$
Expand Down Expand Up @@ -463,7 +469,8 @@ public void onCompletion(ResultsFuture<Boolean> future) {
try {
if (future.get()) {
List<PgColInfo> cols = getPgColInfo(stmt.getResultSet().getMetaData());
client.sendResults(sql, stmt.getResultSet(), cols, completion, -1, true);
String tag = PgBackendProtocol.getCompletionTag(sql, null);
client.sendResults(sql, stmt.getResultSet(), cols, completion, -1, tag.equals("SELECT") || tag.equals("SHOW") ); //$NON-NLS-1$ //$NON-NLS-2$
} else {
client.sendUpdateCount(sql, stmt.getUpdateCount());
setEncoding();
Expand Down Expand Up @@ -778,6 +785,14 @@ else if (sql.equalsIgnoreCase("select current_schema()")) { //$NON-NLS-1$
else if (sql.equals("SELECT typinput='array_in'::regproc, typtype FROM pg_catalog.pg_type WHERE typname = $1")) { //$NON-NLS-1$
return "SELECT substring(typname,1,1) = '_', typtype FROM pg_catalog.pg_type WHERE typname = ?"; //$NON-NLS-1$
}
if ((m = columnMetadataPattern.matcher(modified)).matches()) {
return "select t1.schemaname as nspname, c.relname, t1.name as attname, t.oid as attypid, t.typname, convert(t1.Position, short) as attnum, t.typlen as attlen," //$NON-NLS-1$
+ PgCatalogMetadataStore.TYPMOD + " as atttypmod, " //$NON-NLS-1$
+ "CASE WHEN (t1.NullType = 'No Nulls') THEN true ELSE false END as attnotnull, c.relhasrules, c.relkind, c.oid, pg_get_expr(case when t1.IsAutoIncremented then 'nextval(' else t1.DefaultValue end, c.oid), " //$NON-NLS-1$
+ " case t.typtype when 'd' then t.typbasetype else 0 end, t.typtypmod, c.relhasoids from sys.columns as t1, pg_catalog.matpg_datatype as t, pg_catalog.pg_class c where c.relnspname=t1.schemaname and c.relname=t1.tablename and t1.DataType = t.Name and c.oid = " //$NON-NLS-1$
+ m.group(1)
+ " order by nspname, relname, attnum"; //$NON-NLS-1$
}
}
else if (sql.equalsIgnoreCase("show max_identifier_length")){ //$NON-NLS-1$
return "select 63"; //$NON-NLS-1$
Expand All @@ -792,10 +807,10 @@ else if ((m = rollbackPattern.matcher(modified)).matches()) {
return "ROLLBACK"; //$NON-NLS-1$
}
else if ((m = savepointPattern.matcher(sql)).matches()) {
return "SELECT 0"; //$NON-NLS-1$
return "SELECT 0 from (select 1) x where 1=0"; //$NON-NLS-1$
}
else if ((m = releasePattern.matcher(sql)).matches()) {
return "SELECT 0"; //$NON-NLS-1$
return "SELECT 0 from (select 1) x where 1=0"; //$NON-NLS-1$
}
for (int i = 0; i < modified.length(); i++) {
switch (modified.charAt(i)) {
Expand Down
33 changes: 10 additions & 23 deletions runtime/src/main/java/org/teiid/transport/PgBackendProtocol.java
View file
Expand Up @@ -20,25 +20,7 @@
* 02110-1301 USA.
*/package org.teiid.transport;

import static org.teiid.odbc.PGUtil.PG_TYPE_BOOL;
import static org.teiid.odbc.PGUtil.PG_TYPE_BPCHAR;
import static org.teiid.odbc.PGUtil.PG_TYPE_BYTEA;
import static org.teiid.odbc.PGUtil.PG_TYPE_CHARARRAY;
import static org.teiid.odbc.PGUtil.PG_TYPE_DATE;
import static org.teiid.odbc.PGUtil.PG_TYPE_FLOAT4;
import static org.teiid.odbc.PGUtil.PG_TYPE_FLOAT8;
import static org.teiid.odbc.PGUtil.PG_TYPE_INT2;
import static org.teiid.odbc.PGUtil.PG_TYPE_INT2VECTOR;
import static org.teiid.odbc.PGUtil.PG_TYPE_INT4;
import static org.teiid.odbc.PGUtil.PG_TYPE_INT8;
import static org.teiid.odbc.PGUtil.PG_TYPE_NUMERIC;
import static org.teiid.odbc.PGUtil.PG_TYPE_OIDARRAY;
import static org.teiid.odbc.PGUtil.PG_TYPE_OIDVECTOR;
import static org.teiid.odbc.PGUtil.PG_TYPE_TEXT;
import static org.teiid.odbc.PGUtil.PG_TYPE_TEXTARRAY;
import static org.teiid.odbc.PGUtil.PG_TYPE_TIME;
import static org.teiid.odbc.PGUtil.PG_TYPE_TIMESTAMP_NO_TMZONE;
import static org.teiid.odbc.PGUtil.PG_TYPE_VARCHAR;
import static org.teiid.odbc.PGUtil.*;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufOutputStream;
import io.netty.buffer.Unpooled;
Expand Down Expand Up @@ -417,10 +399,16 @@ private void sendEmptyQueryResponse() {
@Override
public void sendCommandComplete(String sql, Integer count) {
startMessage('C');
String tag = getCompletionTag(sql, count);
writeString(tag);
sendMessage();
}

public static String getCompletionTag(String sql, Integer count) {
String tag;
if (StringUtil.startsWithIgnoreCase(sql, "BEGIN") || StringUtil.startsWithIgnoreCase(sql, "START TRANSACTION")) {
tag = "BEGIN";
} else if (sql.indexOf(' ') == -1) {
} else if (sql.indexOf(' ') == -1 || sql.equals("CLOSE CURSOR")) {
//should already be a completion tag
tag = sql.toUpperCase();
if (count != null) {
Expand All @@ -433,12 +421,11 @@ public void sendCommandComplete(String sql, Integer count) {
if (tag.equals("EXEC") || tag.equals("CALL")) {
tag = "SELECT";
}
if (count != null) {
if (count != null && !(tag.equalsIgnoreCase("SAVEPOINT") || tag.equalsIgnoreCase("RELEASE"))) {
tag += " " + count;
}
}
writeString(tag);
sendMessage();
return tag;
}

private void sendDataRow(ResultSet rs, List<PgColInfo> cols) throws SQLException, IOException {
Expand Down
2 changes: 2 additions & 0 deletions runtime/src/test/resources/adminapi-test-vdb.xml
View file
Expand Up @@ -3,6 +3,8 @@

<description>The adminapi test VDB</description>

<property name="include-pg-metadata" value="false"/>

<model name="TestModel">
<property name="supports-multi-source-bindings" value="true"/>
<source name="text-connector" translator-name="file" connection-jndi-name="java:/test-file"/>
Expand Down

0 comments on commit c2b270d

Please sign in to comment.