Skip to content

Commit

Permalink
TEIID-3042 adding a usage table to system admin to determine dependency
Browse files Browse the repository at this point in the history 
information
  • Loading branch information
@shawkins
shawkins committed Jul 24, 2014
1 parent 79e6446 commit e68759d
Show file tree
Hide file tree
Showing 31 changed files with 738 additions and 432 deletions.
16 changes: 16 additions & 0 deletions api/src/main/java/org/teiid/metadata/AbstractMetadataRecord.java
View file
Expand Up @@ -24,6 +24,7 @@

import java.io.IOException;
import java.io.Serializable;
import java.util.Collection;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;
Expand All @@ -40,6 +41,8 @@
*/
public abstract class AbstractMetadataRecord implements Serializable {

private static final Collection<AbstractMetadataRecord> EMPTY_INCOMING = Collections.emptyList();

public interface Modifiable {
long getLastModified();
}
Expand All @@ -61,6 +64,8 @@ public interface DataModifiable {

private volatile Map<String, String> properties;
private String annotation;

private Collection<AbstractMetadataRecord> incomingObjects;

public static final String RELATIONAL_URI = "{http://www.teiid.org/ext/relational/2012}"; //$NON-NLS-1$

Expand Down Expand Up @@ -234,5 +239,16 @@ private void readObject(java.io.ObjectInputStream in) throws IOException, ClassN
public int hashCode() {
return getUUID().hashCode();
}

public Collection<AbstractMetadataRecord> getIncomingObjects() {
if (incomingObjects == null) {
return EMPTY_INCOMING;
}
return incomingObjects;
}

public void setIncomingObjects(Collection<AbstractMetadataRecord> incomingObjects) {
this.incomingObjects = incomingObjects;
}

}
85 changes: 81 additions & 4 deletions engine/src/main/java/org/teiid/dqp/internal/process/DataTierManagerImpl.java
View file
Expand Up @@ -80,6 +80,7 @@
import org.teiid.logging.MessageLevel;
import org.teiid.metadata.*;
import org.teiid.metadata.Table.TriggerEvent;
import org.teiid.metadata.Table.Type;
import org.teiid.query.QueryPlugin;
import org.teiid.query.metadata.CompositeMetadataStore;
import org.teiid.query.metadata.CompositeMetadataStore.RecordHolder;
Expand Down Expand Up @@ -193,7 +194,8 @@ private enum SystemAdminTables {
VDBRESOURCES,
TRIGGERS,
VIEWS,
STOREDPROCEDURES
STOREDPROCEDURES,
USAGE
}

private enum SystemAdminProcs {
Expand Down Expand Up @@ -733,12 +735,12 @@ protected void fillRow(List<Object> row, List<?> record,
CommandContext cc, SimpleIterator<List<?>> iter) {
row.add(vdb.getName());
ForeignKey key = (ForeignKey) record.get(0);
Table pkTable = key.getPrimaryKey().getParent();
Table pkTable = key.getReferenceKey().getParent();
Column column = (Column) record.get(1);
Short pos = (Short) record.get(2);
row.add(pkTable.getParent().getName());
row.add(pkTable.getName());
row.add(key.getPrimaryKey().getColumns().get(pos-1).getName());
row.add(key.getReferenceKey().getColumns().get(pos-1).getName());
row.add(vdb.getName());
row.add(key.getParent().getParent().getName());
row.add(key.getParent().getName());
Expand All @@ -747,7 +749,7 @@ protected void fillRow(List<Object> row, List<?> record,
row.add(DatabaseMetaData.importedKeyNoAction);
row.add(DatabaseMetaData.importedKeyNoAction);
row.add(key.getName());
row.add(key.getPrimaryKey().getName());
row.add(key.getReferenceKey().getName());
row.add(DatabaseMetaData.importedKeyInitiallyDeferred);
}

Expand All @@ -764,6 +766,81 @@ protected Collection<List<?>> getChildren(Table parent) {
return cols;
}
});
name = SystemAdminTables.USAGE.name();
columns = getColumns(tm, name);
systemAdminTables.put(SystemAdminTables.USAGE, new ChildRecordExtractionTable<AbstractMetadataRecord, AbstractMetadataRecord>(
new RecordTable<AbstractMetadataRecord>(new int[] {0}, columns.subList(1, 2)) {
@Override
protected void fillRow(AbstractMetadataRecord s,
List<Object> rowBuffer) {
rowBuffer.add(s.getUUID());
}

@Override
public SimpleIterator<AbstractMetadataRecord> processQuery(
VDBMetaData vdb, CompositeMetadataStore metadataStore,
BaseIndexInfo<?> ii) {
return processQuery(vdb, metadataStore.getOids(), ii);
}

@Override
protected AbstractMetadataRecord extractRecord(Object val) {
if (val != null) {
return ((RecordHolder)val).getRecord();
}
return null;
}
}, columns) {

@Override
public void fillRow(List<Object> row, AbstractMetadataRecord entry, VDBMetaData vdb, TransformationMetadata metadata, CommandContext cc, SimpleIterator<AbstractMetadataRecord> iter) {
AbstractMetadataRecord currentParent = ((ExpandingSimpleIterator<AbstractMetadataRecord, AbstractMetadataRecord>)iter).getCurrentParent();
row.add(vdb.getName());
row.add(currentParent.getUUID());
row.add(getType(currentParent));
row.add(currentParent.getParent().getName());
row.add(currentParent.getName());
row.add(null); //column usage not yet supported
row.add(entry.getUUID());
row.add(getType(entry));
if (entry instanceof Column) {
row.add(entry.getParent().getParent().getName());
row.add(entry.getParent().getName());
row.add(entry.getName());
} else {
row.add(entry.getParent().getName());
row.add(entry.getName());
row.add(null);
}
}

private String getType(AbstractMetadataRecord record) {
if (record instanceof Table) {
Table t = (Table)record;
if (t.getTableType() == Type.Table && t.isVirtual()) {
//TODO: this change should be on the Table object as well
return "View"; //$NON-NLS-1$
}
return t.getTableType().name();
}
if (record instanceof Procedure) {
Procedure p = (Procedure)record;
if (p.isFunction()) {
return p.getType().name();
}
if (p.isVirtual()) {
return "StoredProcedure"; //$NON-NLS-1$
}
return "ForeignProcedure"; //$NON-NLS-1$
}
return record.getClass().getSimpleName();
}

@Override
protected Collection<AbstractMetadataRecord> getChildren(AbstractMetadataRecord parent) {
return parent.getIncomingObjects();
}
});
}

private List<ElementSymbol> getColumns(TransformationMetadata tm,
Expand Down
80 changes: 71 additions & 9 deletions engine/src/main/java/org/teiid/query/metadata/MetadataValidator.java
View file
Expand Up @@ -21,9 +21,11 @@
*/
package org.teiid.query.metadata;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
Expand All @@ -32,7 +34,9 @@
import org.teiid.adminapi.impl.ModelMetaData;
import org.teiid.adminapi.impl.ModelMetaData.Message.Severity;
import org.teiid.adminapi.impl.VDBMetaData;
import org.teiid.api.exception.query.QueryParserException;
import org.teiid.api.exception.query.QueryResolverException;
import org.teiid.core.TeiidComponentException;
import org.teiid.core.TeiidException;
import org.teiid.core.types.DataTypeManager;
import org.teiid.language.SQLConstants;
Expand All @@ -54,10 +58,13 @@
import org.teiid.query.sql.lang.Command;
import org.teiid.query.sql.lang.QueryCommand;
import org.teiid.query.sql.navigator.PreOrPostOrderNavigator;
import org.teiid.query.sql.symbol.ElementSymbol;
import org.teiid.query.sql.symbol.Expression;
import org.teiid.query.sql.symbol.GroupSymbol;
import org.teiid.query.sql.symbol.Symbol;
import org.teiid.query.sql.visitor.ElementCollectorVisitor;
import org.teiid.query.sql.visitor.EvaluatableVisitor;
import org.teiid.query.sql.visitor.GroupCollectorVisitor;
import org.teiid.query.sql.visitor.ValueIteratorProviderCollectorVisitor;
import org.teiid.query.validator.Validator;
import org.teiid.query.validator.ValidatorFailure;
Expand Down Expand Up @@ -274,13 +281,15 @@ private void validate(VDBMetaData vdb, ModelMetaData model, AbstractMetadataReco
Command command = parser.parseProcedure(p.getQueryPlan(), false);
QueryResolver.resolveCommand(command, new GroupSymbol(p.getFullName()), Command.TYPE_STORED_PROCEDURE, metadata, false);
resolverReport = Validator.validate(command, metadata);
determineDependencies(p, command);
} else if (record instanceof Table) {
Table t = (Table)record;

GroupSymbol symbol = new GroupSymbol(t.getFullName());
ResolverUtil.resolveGroup(symbol, metadata);
if (t.isVirtual() && (t.getColumns() == null || t.getColumns().isEmpty())) {
QueryCommand command = (QueryCommand)QueryParser.getQueryParser().parseCommand(t.getSelectTransformation());
String selectTransformation = t.getSelectTransformation();
if (t.isVirtual() && (t.getColumns() == null || t.getColumns().isEmpty())) {
QueryCommand command = (QueryCommand)QueryParser.getQueryParser().parseCommand(selectTransformation);
QueryResolver.resolveCommand(command, metadata);
resolverReport = Validator.validate(command, metadata);
if(!resolverReport.hasItems()) {
Expand All @@ -293,6 +302,16 @@ private void validate(VDBMetaData vdb, ModelMetaData model, AbstractMetadataReco
}
}
}
determineDependencies(t, command);
if (t.getInsertPlan() != null && t.isInsertPlanEnabled()) {
validateUpdatePlan(model, report, metadata, t, t.getInsertPlan());
}
if (t.getUpdatePlan() != null && t.isUpdatePlanEnabled()) {
validateUpdatePlan(model, report, metadata, t, t.getUpdatePlan());
}
if (t.getDeletePlan() != null && t.isDeletePlanEnabled()) {
validateUpdatePlan(model, report, metadata, t, t.getDeletePlan());
}
}

boolean addCacheHint = false;
Expand Down Expand Up @@ -330,24 +349,67 @@ private void validate(VDBMetaData vdb, ModelMetaData model, AbstractMetadataReco
}

// this seems to parse, resolve and validate.
QueryNode node = QueryResolver.resolveView(symbol, new QueryNode(t.getSelectTransformation()), SQLConstants.Reserved.SELECT, metadata);
QueryNode node = QueryResolver.resolveView(symbol, new QueryNode(selectTransformation), SQLConstants.Reserved.SELECT, metadata);
CacheHint cacheHint = node.getCommand().getCacheHint();
Long ttl = -1L;
if (cacheHint != null && cacheHint.getTtl() != null && addCacheHint) {
ttl = cacheHint.getTtl();
t.setProperty("{http://www.teiid.org/ext/relational/2012}MATVIEW_TTL", String.valueOf(ttl)); //$NON-NLS-1$
t.setProperty(AbstractMetadataRecord.RELATIONAL_URI + "MATVIEW_TTL", String.valueOf(ttl)); //$NON-NLS-1$
}
}
if(resolverReport != null && resolverReport.hasItems()) {
for (ValidatorFailure v:resolverReport.getItems()) {
log(report, model, v.getStatus() == ValidatorFailure.Status.ERROR?Severity.ERROR:Severity.WARNING, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID31080, record.getFullName(), v.getMessage()));
}
}
processReport(model, record, report, resolverReport);
} catch (TeiidException e) {
log(report, model, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID31080, record.getFullName(), e.getMessage()));
}
}

public static void determineDependencies(AbstractMetadataRecord p, Command command) {
Collection<GroupSymbol> groups = GroupCollectorVisitor.getGroupsIgnoreInlineViews(command, true);
LinkedHashSet<AbstractMetadataRecord> values = new LinkedHashSet<AbstractMetadataRecord>();
for (GroupSymbol group : groups) {
Object mid = group.getMetadataID();
if (mid instanceof TempMetadataAdapter) {
mid = ((TempMetadataID)mid).getOriginalMetadataID();
}
if (mid instanceof AbstractMetadataRecord) {
values.add((AbstractMetadataRecord)mid);
}
}
Collection<ElementSymbol> elems = ElementCollectorVisitor.getElements(command, true, true);
for (ElementSymbol elem : elems) {
Object mid = elem.getMetadataID();
if (mid instanceof TempMetadataAdapter) {
mid = ((TempMetadataID)mid).getOriginalMetadataID();
}
if (mid instanceof AbstractMetadataRecord) {
values.add((AbstractMetadataRecord)mid);
}
}
p.setIncomingObjects(new ArrayList<AbstractMetadataRecord>(values));
}

private void validateUpdatePlan(ModelMetaData model,
ValidatorReport report,
QueryMetadataInterface metadata,
Table t, String plan) throws QueryParserException, QueryResolverException,
TeiidComponentException {
QueryCommand command = (QueryCommand)QueryParser.getQueryParser().parseCommand(plan);
QueryResolver.resolveCommand(command, metadata);
//determineDependencies(t, command); -- these should be tracked against triggers
ValidatorReport resolverReport = Validator.validate(command, metadata);
processReport(model, t, report, resolverReport);
}

private void processReport(ModelMetaData model,
AbstractMetadataRecord record, ValidatorReport report,
ValidatorReport resolverReport) {
if(resolverReport != null && resolverReport.hasItems()) {
for (ValidatorFailure v:resolverReport.getItems()) {
log(report, model, v.getStatus() == ValidatorFailure.Status.ERROR?Severity.ERROR:Severity.WARNING, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID31080, record.getFullName(), v.getMessage()));
}
}
}

private Column addColumn(String name, Class<?> type, Table table, MetadataFactory mf) throws TranslatorException {
if (type == null) {
throw new TranslatorException(QueryPlugin.Util.gs(QueryPlugin.Event.TEIID31086, name, table.getFullName()));
Expand Down
34 changes: 30 additions & 4 deletions engine/src/main/java/org/teiid/query/processor/DdlPlan.java
View file
Expand Up @@ -34,6 +34,7 @@
import org.teiid.common.buffer.BufferManager;
import org.teiid.common.buffer.TupleBatch;
import org.teiid.core.TeiidComponentException;
import org.teiid.core.TeiidException;
import org.teiid.core.TeiidProcessingException;
import org.teiid.core.TeiidRuntimeException;
import org.teiid.dqp.internal.process.DQPWorkContext;
Expand All @@ -43,13 +44,17 @@
import org.teiid.metadata.Table;
import org.teiid.metadata.Table.TriggerEvent;
import org.teiid.query.QueryPlugin;
import org.teiid.query.metadata.MetadataValidator;
import org.teiid.query.metadata.TransformationMetadata;
import org.teiid.query.parser.QueryParser;
import org.teiid.query.resolver.QueryResolver;
import org.teiid.query.sql.LanguageVisitor;
import org.teiid.query.sql.lang.AlterProcedure;
import org.teiid.query.sql.lang.AlterTrigger;
import org.teiid.query.sql.lang.AlterView;
import org.teiid.query.sql.lang.Command;
import org.teiid.query.sql.lang.StoredProcedure;
import org.teiid.query.sql.symbol.GroupSymbol;
import org.teiid.query.util.CommandContext;

public class DdlPlan extends ProcessorPlan {
Expand Down Expand Up @@ -123,17 +128,38 @@ public void visit(AlterTrigger obj) {
}

public static void alterView(VDBMetaData vdb, Table t, String sql) {
TransformationMetadata metadata = vdb.getAttachment(TransformationMetadata.class);

try {
Command command = QueryParser.getQueryParser().parseCommand(t.getSelectTransformation());
QueryResolver.resolveCommand(command, metadata);
MetadataValidator.determineDependencies(t, command);
} catch (TeiidException e) {
//should have been caught in validation, but this logic
//is also not mature so since there is no lock on the vdb
//it is possible that the plan is no longer valid at this point due
//to a concurrent execution
}
t.setSelectTransformation(sql);
t.setLastModified(System.currentTimeMillis());
TransformationMetadata indexMetadata = vdb.getAttachment(TransformationMetadata.class);
indexMetadata.addToMetadataCache(t, "transformation/"+SQLConstants.Reserved.SELECT, null); //$NON-NLS-1$
metadata.addToMetadataCache(t, "transformation/"+SQLConstants.Reserved.SELECT, null); //$NON-NLS-1$
}

public static void alterProcedureDefinition(VDBMetaData vdb, Procedure p, String sql) {
TransformationMetadata metadata = vdb.getAttachment(TransformationMetadata.class);
try {
Command command = QueryParser.getQueryParser().parseProcedure(p.getQueryPlan(), false);
QueryResolver.resolveCommand(command, new GroupSymbol(p.getFullName()), Command.TYPE_STORED_PROCEDURE, metadata, false);
MetadataValidator.determineDependencies(p, command);
} catch (TeiidException e) {
//should have been caught in validation, but this logic
//is also not mature so since there is no lock on the vdb
//it is possible that the plan is no longer valid at this point due
//to a concurrent execution
}
p.setQueryPlan(sql);
p.setLastModified(System.currentTimeMillis());
TransformationMetadata indexMetadata = vdb.getAttachment(TransformationMetadata.class);
indexMetadata.addToMetadataCache(p, "transformation/"+StoredProcedure.class.getSimpleName().toUpperCase(), null); //$NON-NLS-1$
metadata.addToMetadataCache(p, "transformation/"+StoredProcedure.class.getSimpleName().toUpperCase(), null); //$NON-NLS-1$
}

public static void alterInsteadOfTrigger(VDBMetaData vdb, Table t,
Expand Down

0 comments on commit e68759d

Please sign in to comment.