Feat/target delta lake streaming tables

forward_engineering/alterScript/alterScriptFromDeltaHelper.js

@@ -29,6 +29,7 @@ const {
 	getContainerName,
 	replaceSpaceWithUnderscore,
 	prepareName,
+	executeUnlessStreaming,
 } = require('../utils/general');
 const { getModifyPkConstraintsScripts } = require('./alterScriptHelpers/entityHelpers/primaryKeyHelper');
 const { getAlterRelationshipsScriptDtos } = require('./alterScriptHelpers/alterRelationshipsHelper');
@@ -170,7 +171,13 @@ const getAlterCollectionsScriptDtos = ({ schema, definitions, provider, data, ap
 
 		if (getDBVersionNumber(dbVersion) >= Runtime.RUNTIME_SUPPORTING_PK_FK_CONSTRAINTS) {
 			modifiedCollectionPrimaryKeysScriptDtos = getItems(schema, 'entities', 'modified').flatMap(collection => {
-				const scripts = getModifyPkConstraintsScripts(provider)({ collection, dbVersion });
+				const isStreaming = collection?.role?.streamingTable;
+
+				const scripts = executeUnlessStreaming(
+					isStreaming,
+					() => getModifyPkConstraintsScripts(provider)({ collection, dbVersion }),
+					[],
+				);
 
 				return wrapWithUseSchema(getSchemaName(collection), scripts);
 			});

forward_engineering/alterScript/alterScriptHelpers/alterEntityHelper.js

@@ -11,6 +11,7 @@ const {
 	getContainerName,
 	isSupportUnityCatalog,
 	isSupportNotNullConstraints,
+	executeUnlessStreaming,
 } = require('../../utils/general');
 const { getModifyCollectionCommentsScripts } = require('./entityHelpers/commentsHelper');
 const { getCheckConstraintsScriptDtos } = require('./columnHelpers/checkConstraintHelper');
@@ -150,11 +151,19 @@ const getDeleteColumnsScripts = (app, definitions, provider, dbVersion) => entit
 	const properties = getEntityProperties(entity);
 	const columnStatement = getColumnsString(Object.keys(columns));
 	const fullCollectionName = generateFullEntityName({ entity, dbVersion });
+
+	const isStreaming = entity?.role?.streamingTable;
+
 	const { hydratedAddIndex, hydratedDropIndex } = hydrateIndex({ entity, properties, definitions, dbVersion });
 	const modifyScript = generateModifyCollectionScript(app)(entity, definitions, provider, dbVersion);
 	const dropIndexScript = provider.dropTableIndex(hydratedDropIndex);
 	const addIndexScript = getIndexes(...hydratedAddIndex);
-	const deleteColumnScript = provider.dropTableColumns({ name: fullCollectionName, columns: columnStatement });
+
+	const deleteColumnScript = executeUnlessStreaming(
+		isStreaming,
+		() => provider.dropTableColumns({ name: fullCollectionName, columns: columnStatement }),
+		'',
+	);
 
 	const dropIndexScriptDto = AlterScriptDto.getInstance([dropIndexScript], true, true);
 	const addIndexScriptDto = AlterScriptDto.getInstance([addIndexScript], true, false);
@@ -201,7 +210,14 @@ const getModifyColumnsScripts = (app, definitions, ddlProvider, dbVersion) => co
 		},
 	};
 	const hydratedAlterColumnName = hydrateAlterColumnName({ entity: collection, properties, dbVersion });
-	const alterColumnScripts = ddlProvider.alterTableColumnName(hydratedAlterColumnName);
+
+	const isStreaming = collection?.role?.streamingTable;
+	const alterColumnScripts = executeUnlessStreaming(
+		isStreaming,
+		() => ddlProvider.alterTableColumnName(hydratedAlterColumnName),
+		[],
+	);
+
 	const modifiedScript = generateModifyCollectionScript(app)(entityData, definitions, ddlProvider, dbVersion);
 	const { hydratedAddIndex, hydratedDropIndex } = hydrateIndex({
 		entity: collection,
@@ -216,15 +232,32 @@ const getModifyColumnsScripts = (app, definitions, ddlProvider, dbVersion) => co
 		collection,
 		dbVersion,
 	});
-	const modifyNotNullConstraintsScriptDtos = getModifyNonNullColumnsScriptDtos(ddlProvider)({
-		collection,
-		dbVersion,
-	});
-	const modifyCheckConstraintsScriptDtos = getCheckConstraintsScriptDtos(ddlProvider)({ collection, dbVersion });
-	const modifiedDefaultColumnValueScriptDtos = getModifiedDefaultColumnValueScriptDtos(ddlProvider)({
-		collection,
-		dbVersion,
-	});
+
+	const modifyNotNullConstraintsScriptDtos = executeUnlessStreaming(
+		isStreaming,
+		() =>
+			getModifyNonNullColumnsScriptDtos(ddlProvider)({
+				collection,
+				dbVersion,
+			}),
+		[],
+	);
+
+	const modifyCheckConstraintsScriptDtos = executeUnlessStreaming(
+		isStreaming,
+		() => getCheckConstraintsScriptDtos(ddlProvider)({ collection, dbVersion }),
+		[],
+	);
+
+	const modifiedDefaultColumnValueScriptDtos = executeUnlessStreaming(
+		isStreaming,
+		() =>
+			getModifiedDefaultColumnValueScriptDtos(ddlProvider)({
+				collection,
+				dbVersion,
+			}),
+		[],
+	);
 
 	const dropIndexScriptDto = AlterScriptDto.getInstance([dropIndexScript], true, true);
 	const addIndexScriptDto = AlterScriptDto.getInstance([addIndexScript], true, false);
@@ -240,7 +273,11 @@ const getModifyColumnsScripts = (app, definitions, ddlProvider, dbVersion) => co
 		return [dropIndexScriptDto, addIndexScriptDto].filter(Boolean);
 	}
 
-	const updateTypeScriptDtos = getUpdateTypesScriptDtos(ddlProvider)(collection, definitions, dbVersion);
+	const updateTypeScriptDtos = executeUnlessStreaming(
+		isStreaming,
+		() => getUpdateTypesScriptDtos(ddlProvider)(collection, definitions, dbVersion),
+		[],
+	);
 
 	return [
 		dropIndexScriptDto,
@@ -272,8 +309,15 @@ const getModifyColumnsScriptsForOlderRuntime = (app, definitions, ddlProvider, d
 			properties: Object.fromEntries(unionProperties),
 		},
 	};
+
+	const isStreaming = collection?.role?.streamingTable;
 	const hydratedAlterColumnName = hydrateAlterColumnName({ entity: collection, properties, dbVersion });
-	const alterColumnScripts = ddlProvider.alterTableColumnName(hydratedAlterColumnName);
+	const alterColumnScripts = executeUnlessStreaming(
+		isStreaming,
+		() => ddlProvider.alterTableColumnName(hydratedAlterColumnName),
+		[],
+	);
+
 	const modifiedScript = generateModifyCollectionScript(app)(entityData, definitions, ddlProvider, dbVersion);
 	const { hydratedAddIndex, hydratedDropIndex } = hydrateIndex({
 		entity: collection,
@@ -293,7 +337,12 @@ const getModifyColumnsScriptsForOlderRuntime = (app, definitions, ddlProvider, d
 		collection,
 		dbVersion,
 	});
-	const modifyCheckConstraintsScriptDtos = getCheckConstraintsScriptDtos(ddlProvider)({ collection, dbVersion });
+
+	const modifyCheckConstraintsScriptDtos = executeUnlessStreaming(
+		isStreaming,
+		() => getCheckConstraintsScriptDtos(ddlProvider)({ collection, dbVersion }),
+		[],
+	);
 
 	let tableModificationScriptDtos = [];
 	if (!_.isEmpty(columnsToDelete)) {

forward_engineering/alterScript/alterScriptHelpers/alterRelationshipsHelper.js

@@ -1,9 +1,11 @@
+const _ = require('lodash');
 const {
 	getFullEntityName,
 	replaceSpaceWithUnderscore,
 	prepareName,
 	getContainerName,
 	replaceDotWithUnderscore,
+	executeUnlessStreaming,
 } = require('../../utils/general');
 const { AlterScriptDto } = require('../types/AlterScriptDto');
 const { getUseSchemaScriptDto } = require('./alterEntityHelper');
@@ -117,13 +119,24 @@ const canRelationshipBeDeleted = relationship => {
 };
 
 /**
- * @return {(deletedRelationships: Array<Object>) => Array<AlterScriptDto>}
+ * @param {Object} ddlProvider - The DDL provider instance.
+ * @return {(deletedRelationships: Array<Object>, entities: Object.<string, Object>) => Array<{isActivated: boolean, scripts: Array<{script: string, isDropScript: boolean}>}>}
  * */
-const getDeleteForeignKeyScripts = ddlProvider => deletedRelationships => {
+const getDeleteForeignKeyScripts = ddlProvider => (deletedRelationships, entities) => {
 	return deletedRelationships
 		.filter(relationship => canRelationshipBeDeleted(relationship))
 		.map(relationship => {
-			const script = getDeleteSingleForeignKeyScript(ddlProvider)(relationship);
+			const childId = relationship?.role?.childCollection;
+			const childTable = entities[childId];
+
+			const isStreaming = childTable?.role?.streamingTable;
+
+			const script = executeUnlessStreaming(
+				isStreaming,
+				() => getDeleteSingleForeignKeyScript(ddlProvider)(relationship),
+				'',
+			);
+
 			return {
 				isActivated: Boolean(relationship.role?.compMod?.isActivated?.new),
 				scripts: [
@@ -162,6 +175,14 @@ const getModifyForeignKeyScript = ddlProvider => relationship => {
 const getAlterRelationshipsScriptDtos = ({ schema, ddlProvider, initialSchemaName }) => {
 	let currentSchemaName = initialSchemaName;
 
+	const allEntitiesArray = [
+		...getItems(schema, 'entities', 'added'),
+		...getItems(schema, 'entities', 'modified'),
+		...getItems(schema, 'entities', 'deleted'),
+	];
+
+	const allEntities = _.keyBy(allEntitiesArray, item => item?.role?.id);
+
 	const generateAddFkScriptDtos = (addedRelationships, getScript) => {
 		return addedRelationships.filter(relationship => canRelationshipBeAdded(relationship)).flatMap(getScript);
 	};
@@ -204,7 +225,7 @@ const getAlterRelationshipsScriptDtos = ({ schema, ddlProvider, initialSchemaNam
 	);
 	const modifiedRelationships = getItems(schema, 'relationships', 'modified');
 
-	const deleteFkScripts = getDeleteForeignKeyScripts(ddlProvider)(deletedRelationships);
+	const deleteFkScripts = getDeleteForeignKeyScripts(ddlProvider)(deletedRelationships, allEntities);
 	const addFkScripts = getRelationshipsScriptsWithUseSchema(
 		addedRelationships,
 		generateAddFkScriptDtos,

forward_engineering/alterScript/alterScriptHelpers/columnHelpers/alterUnityTagsHelper.js

@@ -28,10 +28,12 @@ const getSetUnityColumnTagsDtos =
 			return [];
 		}
 
-		const script = ddlProvider.setColumnTags({ tableName, columnName, tags: buildTagPairs(setTags) });
+		const isStreaming = entityData?.role?.streamingTable;
+
+		const script = ddlProvider.setColumnTags({ tableName, columnName, tags: buildTagPairs(setTags), isStreaming });
 
 		return [AlterScriptDto.getInstance([script], true, false)];
-	}; //
+	};
 
 /**
  * @param ddlProvider {Object}
@@ -53,10 +55,13 @@ const getUnsetUnityColumnTagsScriptsDtosFrom =
 			return [];
 		}
 
+		const isStreaming = entityData?.role?.streamingTable;
+
 		const script = ddlProvider.unsetColumnTags({
 			tableName,
 			columnName,
 			tags: getUnsetTagsNamesParamString({ unsetTags }),
+			isStreaming,
 		});
 
 		return [AlterScriptDto.getInstance([script], true, true)];

forward_engineering/alterScript/alterScriptHelpers/entityHelpers/addColumnsHelper.js

@@ -5,6 +5,7 @@ const {
 	generateFullEntityName,
 	prepareName,
 	getDBVersionNumber,
+	executeUnlessStreaming,
 } = require('../../../utils/general');
 const { getIndexes } = require('../../../helpers/indexHelper');
 const { AlterScriptDto } = require('../../types/AlterScriptDto');
@@ -66,6 +67,8 @@ const getAddColumnsScriptsForModifyModifyCollectionScript = provider => (entity,
 	const entityData = { ...entity, ..._.omit(entity.role, ['properties']) };
 	const { columns } = getColumns(entityData, definitions, dbVersion);
 
+	const isStreaming = entityData?.streamingTable;
+
 	// "NOT NULL" constraint is baked right into the "column statement". We are unsetting "not null" constraint
 	// property on each column so that we could add these constraints in separate statements and not have it duplicated.
 	const columnsWithoutNotNull = getColumnsWithoutNotNullConstraint(columns);
@@ -76,10 +79,17 @@ const getAddColumnsScriptsForModifyModifyCollectionScript = provider => (entity,
 	const { hydratedAddIndex, hydratedDropIndex } = hydrateIndex({ entity, properties, definitions, dbVersion });
 	const dropIndexScript = provider.dropTableIndex(hydratedDropIndex);
 	const addIndexScript = getIndexes(...hydratedAddIndex);
-	const addColumnScript = provider.addTableColumns({ name: fullCollectionName, columns: columnStatement });
+
+	const addColumnScript = executeUnlessStreaming(
+		isStreaming,
+		() => provider.addTableColumns({ name: fullCollectionName, columns: columnStatement }),
+		'',
+	);
 
 	const isUnityTagsSupported = getDBVersionNumber(dbVersion) >= Runtime.MINIMUM_UNITY_TAGS_SUPPORT_VERSION;
-	const columnsUnityTagsScript = isUnityTagsSupported ? getColumnTagsStatement(properties, fullCollectionName) : [];
+	const columnsUnityTagsScript = isUnityTagsSupported
+		? getColumnTagsStatement(properties, fullCollectionName, isStreaming)
+		: [];
 	const addColumnScriptWithUnityTags = isUnityTagsSupported
 		? [addColumnScript, ...columnsUnityTagsScript].join('\n')
 		: addColumnScript;

forward_engineering/alterScript/alterScriptHelpers/entityHelpers/alterUnityTagsHelper.js

@@ -27,11 +27,13 @@ const getSetUnityEntityTagsDtos =
 
 		const setTags = getUnityTagsFromCompMod({ tagsToFilter: newUnityTags, filterBy: oldUnityTags });
 
+		const isStreaming = entityData?.role?.streamingTable;
+
 		if (!setTags.length) {
 			return [];
 		}
 
-		const script = ddlProvider.setEntityTags({ name, tags: buildTagPairs(setTags) });
+		const script = ddlProvider.setEntityTags({ name, tags: buildTagPairs(setTags), isStreaming });
 
 		return [AlterScriptDto.getInstance([script], true, false)];
 	};
@@ -49,11 +51,17 @@ const getUnsetUnityEntityTagsScriptsDtosFrom =
 
 		const unsetTags = getUnityTagsFromCompMod({ tagsToFilter: oldUnityTags, filterBy: newUnityTags });
 
+		const isStreaming = entityData?.role?.streamingTable;
+
 		if (!unsetTags.length) {
 			return [];
 		}
 
-		const script = ddlProvider.unsetEntityTags({ name, tags: getUnsetTagsNamesParamString({ unsetTags }) });
+		const script = ddlProvider.unsetEntityTags({
+			name,
+			tags: getUnsetTagsNamesParamString({ unsetTags }),
+			isStreaming,
+		});
 
 		return [AlterScriptDto.getInstance([script], true, true)];
 	};

forward_engineering/alterScript/alterScriptHelpers/entityHelpers/modifyCollectionScript.js

@@ -11,6 +11,7 @@ const {
 	isSupportUnityCatalog,
 	isSupportNotNullConstraints,
 	checkLiquidClusteringPropertyChanged,
+	executeUnlessStreaming,
 } = require('../../../utils/general');
 const { getTableStatement } = require('../../../helpers/tableHelper');
 const { AlterScriptDto } = require('../../types/AlterScriptDto');
@@ -173,20 +174,54 @@ const getModifyCollectionScriptDtos =
 		const compMod = _.get(collection, 'role.compMod', {});
 		const fullCollectionName = generateFullEntityName({ entity: collection, dbVersion });
 
-		const alterTableNameScript = ddlProvider.alterTableName(hydrateAlterTableName(compMod));
+		const isStreaming = collection?.role?.streamingTable;
+
+		const alterTableNameScript = executeUnlessStreaming(
+			isStreaming,
+			() => ddlProvider.alterTableName(hydrateAlterTableName(compMod)),
+			'',
+		);
+
 		const hydratedSerDeProperties = hydrateSerDeProperties(compMod, fullCollectionName);
-		const checkConstraintsDtos = getModifyCheckConstraintsScriptDtos(ddlProvider)(fullCollectionName, collection);
-		const tablePropertiesScriptDtos = getModifiedTablePropertiesScriptDtos(ddlProvider)({ collection, dbVersion });
-		const serDeProperties = ddlProvider.alterSerDeProperties(hydratedSerDeProperties);
-		const modifyLocationScriptDto = getModifyLocationScriptDto(app, ddlProvider)({ collection, dbVersion });
+
+		const checkConstraintsDtos = executeUnlessStreaming(
+			isStreaming,
+			() => getModifyCheckConstraintsScriptDtos(ddlProvider)(fullCollectionName, collection),
+			[],
+		);
+
+		const tablePropertiesScriptDtos = executeUnlessStreaming(
+			isStreaming,
+			() => getModifiedTablePropertiesScriptDtos(ddlProvider)({ collection, dbVersion }),
+			[],
+		);
+
+		const serDeProperties = executeUnlessStreaming(
+			isStreaming,
+			() => ddlProvider.alterSerDeProperties(hydratedSerDeProperties),
+			'',
+		);
+
+		const modifyLocationScriptDto = executeUnlessStreaming(
+			isStreaming,
+			() => getModifyLocationScriptDto(app, ddlProvider)({ collection, dbVersion }),
+			undefined,
+		);
+
 		const unityEntityTagsDtos = getModifyUnityEntityTagsScriptDtos({ ddlProvider })({
 			entityData: collection,
 			name: fullCollectionName,
 		});
-		const checkLiquidClusteringScriptDtos = getModifyClusteringScriptDto({ ddlProvider })({
-			collection,
-			dbVersion,
-		});
+
+		const checkLiquidClusteringScriptDtos = executeUnlessStreaming(
+			isStreaming,
+			() =>
+				getModifyClusteringScriptDto({ ddlProvider })({
+					collection,
+					dbVersion,
+				}),
+			undefined,
+		);
 
 		return {
 			type: 'modify',