Skip to content

Commit

Permalink
TEIID-5356: Infer field metadata from the source columns (logic to in…
Browse files Browse the repository at this point in the history 
…fer metadata, rather than use defaults for views)
  • Loading branch information
@shawkins @johnathonlee
shawkins authored and johnathonlee committed Dec 11, 2018
1 parent 7e1c181 commit 56e1c2d
Show file tree
Hide file tree
Showing 3 changed files with 171 additions and 17 deletions.
78 changes: 63 additions & 15 deletions engine/src/main/java/org/teiid/dqp/internal/process/MetaDataProcessor.java
View file
Expand Up @@ -44,6 +44,7 @@
import org.teiid.dqp.internal.process.DQPWorkContext.Version;
import org.teiid.dqp.internal.process.SessionAwareCache.CacheID;
import org.teiid.dqp.message.RequestID;
import org.teiid.metadata.BaseColumn.NullType;
import org.teiid.metadata.Column;
import org.teiid.query.QueryPlugin;
import org.teiid.query.function.FunctionLibrary;
Expand All @@ -60,9 +61,7 @@
import org.teiid.query.sql.lang.SetQuery;
import org.teiid.query.sql.lang.SetQuery.Operation;
import org.teiid.query.sql.lang.StoredProcedure;
import org.teiid.query.sql.symbol.*;
import org.teiid.query.sql.symbol.AggregateSymbol.Type;
import org.teiid.query.sql.symbol.Constant;
import org.teiid.query.sql.symbol.ElementSymbol;
import org.teiid.query.sql.symbol.Expression;
import org.teiid.query.sql.symbol.Function;
Expand Down Expand Up @@ -92,6 +91,8 @@ public class MetaDataProcessor {
private RequestID requestID;

private boolean labelAsName;

private boolean useJDBCDefaultPrecision = true;

private boolean useJDBCDefaultPrecision = true;

Expand Down Expand Up @@ -143,6 +144,17 @@ MetadataResult processMessage(RequestID requestId, DQPWorkContext workContext, S
return obtainMetadataForPreparedSql(preparedSql, workContext, allowDoubleQuotedVariable);
}

private Map<Integer, Object> createXMLColumnMetadata(Query xmlCommand) {
GroupSymbol doc = xmlCommand.getFrom().getGroups().get(0);
Map<Integer, Object> xmlMetadata = getDefaultColumn(doc.getName(), XML_COLUMN_NAME, XMLType.class);
// Override size as XML may be big
xmlMetadata.put(ResultsMetadataConstants.DISPLAY_SIZE, JDBCSQLTypeInfo.XML_COLUMN_LENGTH);
MetaDataProcessor mdp = new MetaDataProcessor(null, null, "", 1);

return xmlMetadata;
}


// For each projected symbol, construct a metadata map
private MetadataResult getMetadataForCommand(Command originalCommand) throws TeiidComponentException {
Map<Integer, Object>[] columnMetadata = null;
Expand Down Expand Up @@ -242,9 +254,7 @@ private Map<Integer, Object>[] createProjectedSymbolMetadata(Command originalCom
for(int i=0; symbolIter.hasNext(); i++) {
Expression symbol = symbolIter.next();
String shortColumnName = Symbol.getShortName(Symbol.getOutputName(symbol));
if(symbol instanceof AliasSymbol) {
symbol = ((AliasSymbol)symbol).getSymbol();
}
symbol = SymbolMap.getExpression(symbol);
try {
columnMetadata[i] = createColumnMetadata(shortColumnName, symbol);
} catch(QueryMetadataException e) {
Expand Down Expand Up @@ -305,14 +315,45 @@ private MetadataResult obtainMetadataForPreparedSql(String sql, DQPWorkContext w
return getMetadataForCommand(command);
}

private Map<Integer, Object> createXMLColumnMetadata(Query xmlCommand) {
GroupSymbol doc = xmlCommand.getFrom().getGroups().get(0);
Map<Integer, Object> xmlMetadata = getDefaultColumn(doc.getName(), XML_COLUMN_NAME, XMLType.class);

// Override size as XML may be big
xmlMetadata.put(ResultsMetadataConstants.DISPLAY_SIZE, JDBCSQLTypeInfo.XML_COLUMN_LENGTH);
/**
* Set the easily determined metadata from symbol on the given Column
* @param column
* @param symbol
* @param metadata
* @throws QueryMetadataException
* @throws TeiidComponentException
*/
public static void setColumnMetadata(Column column, Expression symbol, QueryMetadataInterface metadata) throws QueryMetadataException, TeiidComponentException {
//do a dummy initialization of the metadataprocessor
String empty = ""; //$NON-NLS-1$
MetaDataProcessor mdp = new MetaDataProcessor(null, null, empty, 1);
mdp.metadata = metadata;
mdp.useJDBCDefaultPrecision = false;
Map<Integer, Object> metadataMap = mdp.createColumnMetadata(empty, symbol);

return xmlMetadata;
//set the fields from the column metadata
column.setCaseSensitive(Boolean.TRUE.equals(metadataMap.get(ResultsMetadataConstants.CASE_SENSITIVE)));
column.setCurrency(Boolean.TRUE.equals(metadataMap.get(ResultsMetadataConstants.CURRENCY)));
Object nullable = metadataMap.get(ResultsMetadataConstants.NULLABLE);
if (nullable == ResultsMetadataConstants.NULL_TYPES.NOT_NULL) {
column.setNullType(NullType.No_Nulls);
} else if (nullable == ResultsMetadataConstants.NULL_TYPES.NULLABLE) {
column.setNullType(NullType.Nullable);
}
Integer val = (Integer)metadataMap.get(ResultsMetadataConstants.PRECISION);
if (val != null) {
column.setPrecision(val);
column.setLength(val);
}
val = (Integer)metadataMap.get(ResultsMetadataConstants.RADIX);
if (val != null) {
column.setRadix(val);
}
val = (Integer)metadataMap.get(ResultsMetadataConstants.SCALE);
if (val != null) {
column.setScale(val);
}
column.setSigned(Boolean.TRUE.equals(metadataMap.get(ResultsMetadataConstants.SIGNED)));
}

private Map<Integer, Object> createColumnMetadata(String label, Expression symbol) throws QueryMetadataException, TeiidComponentException {
Expand Down Expand Up @@ -451,7 +492,10 @@ private int getColumnPrecision(Class<?> dataType, Object elementID) throws Query
return precision;
}
}
return JDBCSQLTypeInfo.getDefaultPrecision(dataType).intValue();
if (useJDBCDefaultPrecision) {
return JDBCSQLTypeInfo.getDefaultPrecision(dataType).intValue();
}
return 0;
}

/**
Expand Down Expand Up @@ -529,10 +573,14 @@ public Map<Integer, Object> getDefaultColumn(String tableName, String columnName
column.put(ResultsMetadataConstants.CURRENCY, Boolean.FALSE);
column.put(ResultsMetadataConstants.DATA_TYPE, DataTypeManager.getDataTypeName(javaType));
column.put(ResultsMetadataConstants.RADIX, JDBCSQLTypeInfo.DEFAULT_RADIX);
column.put(ResultsMetadataConstants.SCALE, JDBCSQLTypeInfo.DEFAULT_SCALE);
column.put(ResultsMetadataConstants.SIGNED, Boolean.TRUE);

if (useJDBCDefaultPrecision) {
column.put(ResultsMetadataConstants.PRECISION, JDBCSQLTypeInfo.getDefaultPrecision(javaType));
column.put(ResultsMetadataConstants.SCALE, JDBCSQLTypeInfo.DEFAULT_SCALE);
}
//otherwise do not set precision and scale explicitly as we have default logic around the type

column.put(ResultsMetadataConstants.PRECISION, JDBCSQLTypeInfo.getDefaultPrecision(javaType));
column.put(ResultsMetadataConstants.DISPLAY_SIZE, JDBCSQLTypeInfo.getMaxDisplaySize(javaType));
return column;
}
Expand Down
32 changes: 30 additions & 2 deletions engine/src/main/java/org/teiid/query/metadata/MetadataValidator.java
View file
Expand Up @@ -34,6 +34,7 @@
import org.teiid.adminapi.impl.ModelMetaData;
import org.teiid.adminapi.impl.ModelMetaData.Message.Severity;
import org.teiid.adminapi.impl.VDBMetaData;
import org.teiid.api.exception.query.QueryMetadataException;
import org.teiid.api.exception.query.QueryParserException;
import org.teiid.api.exception.query.QueryResolverException;
import org.teiid.core.TeiidComponentException;
Expand Down Expand Up @@ -67,6 +68,7 @@
import org.teiid.query.sql.symbol.Expression;
import org.teiid.query.sql.symbol.GroupSymbol;
import org.teiid.query.sql.symbol.Symbol;
import org.teiid.query.sql.util.SymbolMap;
import org.teiid.query.sql.visitor.ElementCollectorVisitor;
import org.teiid.query.sql.visitor.EvaluatableVisitor;
import org.teiid.query.sql.visitor.GroupCollectorVisitor;
Expand Down Expand Up @@ -324,7 +326,7 @@ private void validate(VDBMetaData vdb, ModelMetaData model, AbstractMetadataReco
List<Expression> symbols = command.getProjectedSymbols();
for (Expression column:symbols) {
try {
addColumn(Symbol.getShortName(column), column.getType(), t, mf);
addColumn(column, t, mf, metadata);
} catch (TranslatorException e) {
log(report, model, e.getMessage());
}
Expand Down Expand Up @@ -471,12 +473,38 @@ private void processReport(ModelMetaData model,
}
}

private Column addColumn(String name, Class<?> type, Table table, MetadataFactory mf) throws TranslatorException {
private Column addColumn(Expression toCopy, Table table, MetadataFactory mf, QueryMetadataInterface metadata) throws TranslatorException, QueryMetadataException, TeiidComponentException {
String name = Symbol.getShortName(toCopy);
Class<?> type = toCopy.getType();
if (type == null) {
throw new TranslatorException(QueryPlugin.Util.gs(QueryPlugin.Event.TEIID31086, name, table.getFullName()));
}
Column column = mf.addColumn(name, DataTypeManager.getDataTypeName(type), table);
column.setUpdatable(table.supportsUpdate());
//determine the column metadata
toCopy = SymbolMap.getExpression(toCopy);
boolean metadataSet = false;
if (toCopy instanceof ElementSymbol) {
Object mid = ((ElementSymbol) toCopy).getMetadataID();
if (mid instanceof Column) {
metadataSet = true;
Column other = (Column)mid;
column.setCaseSensitive(other.isCaseSensitive());
column.setCharOctetLength(other.getCharOctetLength());
column.setCurrency(other.isCurrency());
column.setFixedLength(other.isFixedLength());
column.setFormat(other.getFormat());
column.setLength(other.getLength());
column.setNullType(other.getNullType());
column.setPrecision(other.getPrecision());
column.setRadix(other.getRadix());
column.setScale(other.getScale());
column.setSigned(other.isSigned());
}
}
if (!metadataSet) {
MetaDataProcessor.setColumnMetadata(column, toCopy, metadata);
}
return column;
}

Expand Down
78 changes: 78 additions & 0 deletions engine/src/test/java/org/teiid/query/metadata/TestMetadataValidator.java
View file
Expand Up @@ -32,12 +32,19 @@
import org.teiid.adminapi.Model;
import org.teiid.adminapi.impl.ModelMetaData;
import org.teiid.adminapi.impl.VDBMetaData;
<<<<<<< HEAD
import org.teiid.metadata.BaseColumn.NullType;
import org.teiid.metadata.Column;
import org.teiid.metadata.Column.SearchType;
||||||| parent of b335ba12bc... broken TEIID-5356 logic to infer metadata, rather than use defaults for views
=======
import org.teiid.metadata.BaseColumn.NullType;
>>>>>>> b335ba12bc... broken TEIID-5356 logic to infer metadata, rather than use defaults for views
import org.teiid.metadata.MetadataFactory;
import org.teiid.metadata.MetadataStore;
import org.teiid.metadata.ParseException;
import org.teiid.metadata.Column;
import org.teiid.metadata.Column.SearchType;
import org.teiid.metadata.Table;
import org.teiid.query.function.SystemFunctionManager;
import org.teiid.query.parser.QueryParser;
Expand Down Expand Up @@ -453,6 +460,7 @@ public void testSkipDocumentModel() throws Exception {
ValidatorReport report = new MetadataValidator().validate(this.vdb, this.store);
assertTrue(printError(report), report.hasItems());
}
<<<<<<< HEAD

@Test
public void testViewWithoutColumns() throws Exception {
Expand Down Expand Up @@ -553,6 +561,76 @@ public void testSetQueryViewWithoutColumns() throws Exception {
assertEquals(14, e3.getPrecision());
assertEquals(5, e3.getScale());
}
||||||| parent of b335ba12bc... broken TEIID-5356 logic to infer metadata, rather than use defaults for views
=======

@Test
public void testViewWithoutColumns() throws Exception {
String ddl = "CREATE FOREIGN TABLE G1(\n" +
"e1 integer primary key,\n" +
"e2 varchar(10) unique,\n" +
"e3 date not null unique,\n" +
"e4 decimal(12,3) default 12.2 options (searchable 'unsearchable'),\n" +
"e5 integer auto_increment INDEX OPTIONS (UUID 'uuid', NAMEINSOURCE 'nis', SELECTABLE 'NO'),\n" +
"e6 varchar index default 'hello')\n" +
"OPTIONS (CARDINALITY 12, UUID 'uuid2', UPDATABLE 'true', FOO 'BAR', ANNOTATION 'Test Table'); "
+ " create view v1 as select G1.*, 'a' from G1;";

buildModel("phy1", true, this.vdb, this.store, ddl);

buildTransformationMetadata();

ValidatorReport report = new MetadataValidator().validate(this.vdb, this.store);

assertFalse(printError(report), report.hasItems());

Table table = store.getSchema("phy1").getTable("v1");

assertTrue(table.isVirtual());
assertFalse(table.isSystem());
assertFalse(table.isMaterialized());
assertFalse(table.isDeletePlanEnabled());
assertFalse(table.supportsUpdate());

assertEquals(6, table.getColumns().size());

List<Column> columns = table.getColumns();
Column e1 = columns.get(0);
Column e2 = columns.get(1);
Column e3 = columns.get(2);
Column e4 = columns.get(3);
Column e6 = columns.get(4);
Column e7 = columns.get(5);

assertEquals("e1", e1.getName());
assertEquals("integer", e1.getDatatype().getName());

assertEquals("e2", e2.getName());
assertEquals("string", e2.getDatatype().getName());
assertEquals(NullType.Nullable, e2.getNullType());
assertEquals(10, e2.getLength());
assertEquals(0, e2.getPrecision());

assertEquals("e3", e3.getName());
assertEquals("date", e3.getDatatype().getName());
assertEquals(NullType.No_Nulls, e3.getNullType());

assertEquals("e4", e4.getName());
assertEquals("bigdecimal", e4.getDatatype().getName());
assertEquals(false, e4.isAutoIncremented());
assertEquals(12, e4.getPrecision());
assertEquals(3, e4.getScale());
assertEquals(SearchType.Searchable, e4.getSearchType());
assertEquals(null, e4.getDefaultValue());

assertEquals("e6", e6.getName());
assertEquals("string", e6.getDatatype().getName());

assertEquals("expr2", e7.getName());
assertEquals("string", e7.getDatatype().getName());
assertEquals(4000, e7.getLength());
}
>>>>>>> b335ba12bc... broken TEIID-5356 logic to infer metadata, rather than use defaults for views

private ValidatorReport helpTest(String ddl, boolean expectErrors) throws Exception {
buildModel("pm1", true, this.vdb, this.store, ddl);
Expand Down

0 comments on commit 56e1c2d

Please sign in to comment.