Skip to content

Commit

Permalink
TEIID-4626: Adding a LAZY Refresh Strategy to the materialization loa…
Browse files Browse the repository at this point in the history 
…d based on CDC events
  • Loading branch information
@rareddy
rareddy committed Feb 24, 2017
1 parent d3f2cb9 commit fd1d073
Show file tree
Hide file tree
Showing 32 changed files with 688 additions and 4,889 deletions.
3 changes: 3 additions & 0 deletions engine/src/main/java/org/teiid/query/metadata/MaterializationMetadataRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,12 @@ public class MaterializationMetadataRepository extends MetadataRepository {
public static final String MATVIEW_OWNER_VDB_VERSION = "{http://www.teiid.org/ext/relational/2012}MATVIEW_OWNER_VDB_VERSION"; //$NON-NLS-1$

public static final String MATVIEW_WRITE_THROUGH = "{http://www.teiid.org/ext/relational/2012}MATVIEW_WRITE_THROUGH"; //$NON-NLS-1$
public static final String MATVIEW_REFRESH_TYPE = "{http://www.teiid.org/ext/relational/2012}MATVIEW_REFRESH_TYPE"; //$NON-NLS-1$
public static final String MATVIEW_LAZY_MAX_ALLOWED_STALENESS_PCT = "{http://www.teiid.org/ext/relational/2012}MATVIEW_LAZY_MAX_ALLOWED_STALENESS_PCT"; //$NON-NLS-1$

public enum LoadStates {NEEDS_LOADING, LOADING, LOADED, FAILED_LOAD};
public enum Scope {IMPORTED, FULL};
public enum RefreshType {SNAPSHOT, EAGER, LAZY};
public enum ErrorAction {THROW_EXCEPTION, IGNORE, WAIT}
// Status table column names
//VDBName, VDBVersion, SchemaName, Name, TargetSchemaName, TargetName, Valid, LoadState, Updated, Cardinality, LoadNumber
Expand Down
85 changes: 59 additions & 26 deletions engine/src/main/java/org/teiid/query/metadata/MetadataValidator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -298,8 +298,6 @@ protected void setUUID(AbstractMetadataRecord record) {
}

static class MatViewPropertiesValidator implements MetadataRule {


@Override
public void execute(VDBMetaData vdb, MetadataStore store, ValidatorReport report, MetadataValidator metadataValidator) {

Expand Down Expand Up @@ -354,11 +352,32 @@ public void execute(VDBMetaData vdb, MetadataStore store, ValidatorReport report
}

String scope = t.getProperty(MATVIEW_SHARE_SCOPE, false);
if (scope != null && !scope.equalsIgnoreCase(Scope.IMPORTED.name()) && !scope.equalsIgnoreCase(Scope.FULL.name())) {
metadataValidator.log(report, model, Severity.WARNING, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID31253, t.getFullName(), scope));
if (scope != null && !scope.equalsIgnoreCase(Scope.IMPORTED.name())
&& !scope.equalsIgnoreCase(Scope.FULL.name())) {
metadataValidator.log(report, model, Severity.WARNING,QueryPlugin.Util.gs(QueryPlugin.Event.TEIID31253, t.getFullName(), scope));
t.setProperty(MATVIEW_SHARE_SCOPE, Scope.IMPORTED.name());
}

String refreshType = t.getProperty(MATVIEW_REFRESH_TYPE, false);
if (refreshType == null) {
t.setProperty(MATVIEW_REFRESH_TYPE, RefreshType.SNAPSHOT.name());
} else if (!refreshType.equalsIgnoreCase(RefreshType.LAZY.name())
&& !refreshType.equalsIgnoreCase(RefreshType.EAGER.name())
&& !refreshType.equalsIgnoreCase(RefreshType.SNAPSHOT.name())) {
metadataValidator.log(report, model, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID31253, t.getFullName(), scope));
continue;
}

if (refreshType != null && RefreshType.valueOf(refreshType) == RefreshType.LAZY) {
for (AbstractMetadataRecord st : t.getIncomingObjects()) {
if (st instanceof Table && ((Table)st).isPhysical()) {
addLazyMatViewTrigger(vdb, t, (Table)st, Table.TriggerEvent.INSERT);
addLazyMatViewTrigger(vdb, t, (Table)st, Table.TriggerEvent.UPDATE);
addLazyMatViewTrigger(vdb, t, (Table)st, Table.TriggerEvent.DELETE);
}
}
}

Table statusTable = findTableByName(store, status);
if (statusTable == null) {
metadataValidator.log(report, model, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID31197, t.getFullName(), status));
Expand All @@ -378,6 +397,7 @@ public void execute(VDBMetaData vdb, MetadataStore store, ValidatorReport report
statusTypeMap.put("UPDATED", DataTypeManager.DefaultDataClasses.TIMESTAMP); //$NON-NLS-1$
statusTypeMap.put("LOADNUMBER", DataTypeManager.DefaultDataClasses.LONG); //$NON-NLS-1$
statusTypeMap.put("NODENAME", DataTypeManager.DefaultDataClasses.STRING); //$NON-NLS-1$
statusTypeMap.put("STALECOUNT", DataTypeManager.DefaultDataClasses.LONG); //$NON-NLS-1$

List<Column> statusColumns = statusTable.getColumns();
for(int i = 0 ; i < statusColumns.size() ; i ++) {
Expand Down Expand Up @@ -421,6 +441,19 @@ public void execute(VDBMetaData vdb, MetadataStore store, ValidatorReport report
}
}
}

private void addLazyMatViewTrigger(VDBMetaData vdb, Table t, Table st, Table.TriggerEvent event) {
String name = "on_"+st.getName()+"_"+event.name()+"_for"+t.getName()+"_for_lazymatview";
String plan = "FOR EACH ROW\n"
+ "BEGIN ATOMIC\n"
+ "EXECUTE SYSADMIN.updateStaleCount(schemaName=>'"+t.getParent().getName()+"', viewName=>'"+t.getName()+"');"
+ "END";
Trigger trigger = new Trigger();
trigger.setName(name);
trigger.setEvent(event);
trigger.setPlan(plan);
st.getTriggers().put(name, trigger);
}

private void loadScriptsValidation(VDBMetaData vdb, ValidatorReport report, MetadataValidator metadataValidator, ModelMetaData model, Table matView, String script, String option) {
if(script == null) {
Expand Down Expand Up @@ -549,28 +582,28 @@ private void validate(VDBMetaData vdb, ModelMetaData model, AbstractMetadataReco
addCacheHint = true;
}

// this seems to parse, resolve and validate.
QueryNode node = QueryResolver.resolveView(symbol, new QueryNode(selectTransformation), SQLConstants.Reserved.SELECT, metadata, true);
CacheHint cacheHint = node.getCommand().getCacheHint();
Long ttl = -1L;
if (cacheHint != null && cacheHint.getTtl() != null && addCacheHint
&& t.getProperty(MaterializationMetadataRepository.MATVIEW_TTL, false) == null) {
ttl = cacheHint.getTtl();
t.setProperty(MaterializationMetadataRepository.MATVIEW_TTL, String.valueOf(ttl));
}
if (cacheHint != null && cacheHint.getUpdatable() != null && addCacheHint
&& t.getProperty(MaterializationMetadataRepository.MATVIEW_UPDATABLE, false) == null) {
t.setProperty(MaterializationMetadataRepository.MATVIEW_UPDATABLE, String.valueOf(cacheHint.getUpdatable()));
}
if (cacheHint != null && cacheHint.getPrefersMemory() != null && addCacheHint
&& t.getProperty(MaterializationMetadataRepository.MATVIEW_PREFER_MEMORY, false) == null) {
t.setProperty(MaterializationMetadataRepository.MATVIEW_PREFER_MEMORY, String.valueOf(cacheHint.getPrefersMemory()));
}
if (cacheHint != null && cacheHint.getScope() != null && addCacheHint
&& t.getProperty(MaterializationMetadataRepository.MATVIEW_SHARE_SCOPE, false) == null) {
log(report, model, Severity.WARNING, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID31252, t.getName(), cacheHint.getScope().name()));
t.setProperty(MaterializationMetadataRepository.MATVIEW_SHARE_SCOPE, MaterializationMetadataRepository.Scope.IMPORTED.name());
}
if (addCacheHint && t.isMaterialized()) {
// this seems to parse, resolve and validate.
QueryNode node = QueryResolver.resolveView(symbol, new QueryNode(selectTransformation), SQLConstants.Reserved.SELECT, metadata, true);
CacheHint cacheHint = node.getCommand().getCacheHint();
Long ttl = -1L;
if (cacheHint != null) {
if (cacheHint.getTtl() != null && t.getProperty(MaterializationMetadataRepository.MATVIEW_TTL, false) == null) {
ttl = cacheHint.getTtl();
t.setProperty(MaterializationMetadataRepository.MATVIEW_TTL, String.valueOf(ttl));
}
if (cacheHint.getUpdatable() != null && t.getProperty(MaterializationMetadataRepository.MATVIEW_UPDATABLE, false) == null) {
t.setProperty(MaterializationMetadataRepository.MATVIEW_UPDATABLE, String.valueOf(cacheHint.getUpdatable()));
}
if (cacheHint.getPrefersMemory() != null && t.getProperty(MaterializationMetadataRepository.MATVIEW_PREFER_MEMORY, false) == null) {
t.setProperty(MaterializationMetadataRepository.MATVIEW_PREFER_MEMORY, String.valueOf(cacheHint.getPrefersMemory()));
}
if (cacheHint.getScope() != null && t.getProperty(MaterializationMetadataRepository.MATVIEW_SHARE_SCOPE, false) == null) {
log(report, model, Severity.WARNING, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID31252, t.getName(), cacheHint.getScope().name()));
t.setProperty(MaterializationMetadataRepository.MATVIEW_SHARE_SCOPE, MaterializationMetadataRepository.Scope.IMPORTED.name());
}
}
}
}
processReport(model, record, report, resolverReport);
} catch (TeiidException e) {
Expand Down
Loading

0 comments on commit fd1d073

Please sign in to comment.