load LDM from BIRD website content, and use codes #1345

#1345

bird/birdseed_creator/src/context/context.py

@@ -24,6 +24,7 @@ class Context(object):
     # variables to configure the behaviour
 
     use_codes = True
+    load_ldm = False
     load_eil_from_website = False
     use_variable_long_name = True
     persist_to_regdna = True
@@ -50,6 +51,7 @@ class Context(object):
     #table_parts_to_to_filter_items_map = {}
     table_parts_to_main_catagory_map = {}
     main_catagories_in_scope = []
+    table_part_varaible_transformation_map = {}
 
     # create  regdna  packages
     types_package = ELPackage(name='types')

bird/birdseed_creator/src/entry_points/create_reports_info.py

@@ -15,10 +15,9 @@
 
 
 from entry_points.website_to_sddmodel import RunWebsiteToSDDModel
-from entry_points.sddmodel_to_datamodel import RunSDDModelToDataModel 
+from entry_points.sddmodel_to_datamodel_finrep import RunSDDModelToDataModelFinrep 
 from context.context import Context
 from context.sdd_context import SDDContext
-from process_steps.sddmodel_to_datamodel.translate_sddmodel_to_datamodel import TranslateSDDModelToDataModel
 from persister.persist_to_file import PersistToFile
 from process_steps.report_filters.translate_combinations_to_report_filters import CombinationsToReportFilters
 
@@ -37,7 +36,7 @@ def run(self,context,sdd_context):
     sdd_context.file_directory = '/workspaces/efbt/bird/birdseed_creator/resources'
     sdd_context.output_directory = '/workspaces/efbt/bird/birdseed_creator/results'    
     RunWebsiteToSDDModel().run(sdd_context)
-    RunSDDModelToDataModel().run(context,sdd_context)
+    RunSDDModelToDataModelFinrep().run(context,sdd_context)
     RunCreateReports().run(context,sdd_context)
     persister = PersistToFile()
     persister.save_model_as_regdna_file(context)

bird/birdseed_creator/src/entry_points/sddmodel_to_datamodel_ldm.py

@@ -0,0 +1,55 @@
+# coding=UTF-8#
+# Copyright (c) 2020 Bird Software Solutions Ltd
+# This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License 2.0
+# which accompanies this distribution, and is available at
+# https://www.eclipse.org/legal/epl-2.0/
+#
+# SPDX-License-Identifier: EPL-2.0
+#
+# Contributors:
+#    Neil Mackenzie - initial API and implementation
+#
+'''
+Created on 25 April 2022
+'''
+
+from entry_points.website_to_sddmodel import RunWebsiteToSDDModel 
+from context.context import Context
+from context.sdd_context import SDDContext
+from process_steps.sddmodel_to_datamodel.translate_sddmodel_to_datamodel import TranslateSDDModelToDataModel
+from persister.persist_to_file import PersistToFile
+from utils.ldm_relationship_enricher import LDMRelationshipEnricher
+
+
+class RunSDDModelToDataModelLDM:
+
+    def run(self,context,sdd_context):
+        context.persist_to_regdna = True
+        context.load_eil_from_website  = True
+        context.load_ldm = True
+        TranslateSDDModelToDataModel().do_import(context,sdd_context)
+        LDMRelationshipEnricher.enrich(self,context)
+
+
+
+if __name__ == '__main__':
+    sdd_context = SDDContext()
+    context = Context()
+    context.file_directory = '/workspaces/efbt/bird/birdseed_creator/resources'
+    context.output_directory = '/workspaces/efbt/bird/birdseed_creator/results' 
+    sdd_context.file_directory = '/workspaces/efbt/bird/birdseed_creator/resources'
+    sdd_context.output_directory = '/workspaces/efbt/bird/birdseed_creator/results'   
+    RunWebsiteToSDDModel().run(sdd_context)
+    RunSDDModelToDataModelLDM().run(context,sdd_context)
+    persister = PersistToFile()
+    persister.save_model_as_regdna_file(context)
+    persister.save_model_as_xmi_file(context)
+
+
+
+
+
+
+
+

bird/birdseed_creator/src/persister/persist_to_file.py

@@ -302,6 +302,25 @@ def persist_generation_transformations_to_csv(self, context):
 
                 f.close()
         report_to_table_parts_file.close
+        f = open(context.output_directory + os.sep + 'generations_transformations_csv' +
+                        os.sep + 
+                        'generation_rules_summary.csv', "a",  encoding='utf-8')
+        f.write("Key,Table Part, Main Table, ROL cube Item, Source Table,Source Column,ROL Cube Item,Notes\n")
+
+        for key,value in context.table_part_varaible_transformation_map.items():
+
+            column_name = 'None'
+            table_name= 'None'
+
+            if not(value is None):
+                table_name = value.eContainer().name
+                column_name = value.name
+
+            f.write(key +"," + table_name +","  + column_name +",\n")
+
+
+        f.close()
+
 
     def persist_generation_transformations(self, context):
         '''

bird/birdseed_creator/src/process_steps/sddmodel_to_datamodel/translate_sddmodel_to_datamodel.py

@@ -44,54 +44,73 @@ def add_classes_to_package(self, context,sdd_context):
         '''
         for cube  in sdd_context.cubes.cubes:
             framework = cube.framework_id.framework_id
-            if (framework == context.reporting_framework + "_REF"):
-                class_name = cube.code
+            if not(context.load_ldm):
+                if (framework == context.reporting_framework + "_REF"):
+                    class_name = cube.code
+                else:
+                    if (context.use_codes):
+                        class_name = cube.code
+                    else:
+                        class_name = cube.name
             else:
                 if (context.use_codes):
                     class_name = cube.code
                 else:
                     class_name = cube.name
+
 
             altered_class_name = Utils.make_valid_id(class_name)
             object_id = cube.cube_id
             cube_type = cube.cube_type
             derived = True
 
-            if ((((framework == context.reporting_framework +"_REF") and (cube_type == "RC")) or ( context.load_eil_from_website and ((cube_type == "EIL"))) )):
-
-                altered_class_name = Utils.make_valid_id(class_name)
-                fullName = None
-                if cube_type == "EIL" : 
-                    fullName=altered_class_name
-                else:
-                    fullName=altered_class_name+"_OutputItem"
-
-                if cube_type == "EIL":
-                    derived = False
+            if not(context.load_ldm):
+                if ((((framework == context.reporting_framework +"_REF") and (cube_type == "RC")) or ( context.load_eil_from_website and ((cube_type == "EIL"))) )):
+
+                    altered_class_name = Utils.make_valid_id(class_name)
+                    fullName = None
+                    if cube_type == "EIL" : 
+                        fullName=altered_class_name
+                    else:
+                        fullName=altered_class_name+"_OutputItem"
 
-                eclass = ELClass(name=fullName)
-                eclass.isDerived = derived
-                if context.add_pks_to_input_layer_from_website:
                     if cube_type == "EIL":
-                        pk_name = fullName + "_uniqueID"
-                        attribute = ELAttribute()
-                        attribute.name = pk_name
-                        attribute.eType = context.types.e_string
-                        attribute.eAttributeType = context.types.e_string
-                        attribute.iD = True
-                        attribute.lowerBound = 0
-                        attribute.upperBound = 1
-                        eclass.eStructuralFeatures.append(attribute)
-
-                if not (cube_type == "EIL"):
-                    context.output_tables_package.eClassifiers.extend([
-                                                                       eclass])
-                else:
+                        derived = False
+
+                    eclass = ELClass(name=fullName)
+                    eclass.isDerived = derived
+                    if context.add_pks_to_input_layer_from_website:
+                        if cube_type == "EIL":
+                            pk_name = fullName + "_uniqueID"
+                            attribute = ELAttribute()
+                            attribute.name = pk_name
+                            attribute.eType = context.types.e_string
+                            attribute.eAttributeType = context.types.e_string
+                            attribute.iD = True
+                            attribute.lowerBound = 0
+                            attribute.upperBound = 1
+                            eclass.eStructuralFeatures.append(attribute)
+
+                    if not (cube_type == "EIL"):
+                        context.output_tables_package.eClassifiers.extend([
+                                                                           eclass])
+                    else:
+                        context.input_tables_package.eClassifiers.extend([
+                                                                           eclass])
+
+                    # maintain a map a objectIDs to ELClasses
+                    context.classes_map[object_id] = eclass
+            else:
+                if cube_type == "LDM":
+                    altered_class_name = Utils.make_valid_id(class_name)
+                    fullName = altered_class_name
+                    derived = False
+                    eclass = ELClass(name=fullName)
+                    eclass.isDerived = derived
                     context.input_tables_package.eClassifiers.extend([
-                                                                       eclass])
-
-                # maintain a map a objectIDs to ELClasses
-                context.classes_map[object_id] = eclass
+                                                                           eclass])
+                    # maintain a map a objectIDs to ELClasses
+                    context.classes_map[object_id] = eclass
 
 
     def add_enums_and_literals_to_package_for_input_layer(self, context,sdd_context):

bird/birdseed_creator/src/utils/enrich_ldm_with_il_links_from_fe.py

@@ -0,0 +1,142 @@
+# coding=UTF-8#
+# Copyright (c) 2020 Bird Software Solutions Ltd
+# This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License 2.0
+# which accompanies this distribution, and is available at
+# https://www.eclipse.org/legal/epl-2.0/
+#
+# SPDX-License-Identifier: EPL-2.0
+#
+# Contributors:
+#    Neil Mackenzie - initial API and implementation
+#
+import csv
+from ldm_utils.utils import Utils
+import os
+
+from regdna import ELAttribute, ELClass, ELEnum, ELEnumLiteral, ELOperation, ELReference, ELAnnotation, ELStringToStringMapEntry
+
+class InputLayerLinkEnricher(object):
+    '''
+    After the Forward Engineering process has been run on the LDM, 
+    SQLDevelepor stores information about how whicj column in the Input
+    Layer was created by forward engineering an attribute in the LDM.
+    In SQLdeveloper these are accessed via the 'Impacty analysis'
+    Feature..so we can see what is the equivelent Input Layer column
+    for an LDM attribute.
+    This class is responsable for adding an Annotation to the LDM attribute
+    to show the name of the linked Input Layer column. The name
+    is represented in a 'TableName.ColumnName' format.
+    '''
+
+    def enrich_with_links_to_input_layer_columns(self, context):
+        '''
+        Enrich the attributes of classes of our LDM package with an annotation
+        To show what input layer column is related to LDM attribute.
+        '''
+        InputLayerLinkEnricher.create_attribute_to_column_links(self, context)
+
+    def create_attribute_to_column_links(self, context):
+        file_location = context.file_directory + os.sep + "DM_Mappings.csv"
+        header_skipped = False
+
+        with open(file_location,  encoding='utf-8') as csvfile:
+            filereader = csv.reader(csvfile, delimiter=',', quotechar='"')
+            for row in filereader:
+                if not header_skipped:
+                    header_skipped = True
+                else:
+                    logical_object_name = row[5]
+                    relational_model_name = row[8]
+                    relational_object_Name = row[11]
+                    entity_name = row[12]
+                    table_name = row[13]
+                    if (relational_model_name == context.input_layer_name):
+
+                        # annotate entites
+                        if logical_object_name == entity_name:
+                            ldm_entity = InputLayerLinkEnricher.get_ldm_entity(
+                                self, 
+                                context,
+                                Utils.make_valid_id(entity_name))
+
+                            the_entity_annotation = Utils.get_annotation_with_source(ldm_entity, "il_mapping")
+                            if the_entity_annotation is None: 
+                                the_entity_annotation = ELAnnotation()
+                                the_entity_annotation_directive = Utils.get_annotation_directive(ldm_entity.eContainer(), "il_mapping")
+                                the_entity_annotation.source = the_entity_annotation_directive
+                                ldm_entity.eAnnotations.append(the_entity_annotation)
+
+                            details = the_entity_annotation.details
+
+                            il_tables_count = 0
+
+                            for detail in details:
+                                if detail.key.startswith("il_table"):
+                                    il_tables_count = il_tables_count + 1
+
+                            detail1 = ELStringToStringMapEntry()
+                            if il_tables_count ==0:
+                                detail1.key = "il_table"
+                            else:
+                                detail1.key = "il_table" + str(il_tables_count)
+
+                            detail1.value = table_name
+                            details.append(detail1)
+                        else:
+                            # annotate attributes
+                            ldm_attribute = InputLayerLinkEnricher.get_ldm_attribute(
+                                self, 
+                                context,
+                                Utils.make_valid_id(entity_name),
+                                Utils.make_valid_id(logical_object_name))
+                            # logical_attribute_to_relational_name[ldm_attribute] =  table_name + "." + relational_object_Name
+                            if not(ldm_attribute is None):
+                                if isinstance(ldm_attribute,ELAttribute):
+                                    the_attribute_annotation = Utils.get_annotation_with_source(ldm_attribute, "il_mapping")
+                                    if the_attribute_annotation is None: 
+                                        the_attribute_annotation = ELAnnotation()
+                                        the_attribute_annotation_directive = Utils.get_annotation_directive(ldm_attribute.eContainer().eContainer(), "key")
+                                        the_attribute_annotation.source = the_attribute_annotation_directive
+                                        ldm_attribute.eAnnotations.append(the_attribute_annotation)
+
+                                    details = the_attribute_annotation.details
+                                    detail1 = ELStringToStringMapEntry()
+                                    detail1.key = "il_column"
+                                    detail1.value = table_name + "." + relational_object_Name
+                                    details.append(detail1)
+
+                                if isinstance(ldm_attribute,ELReference):
+                                    the_reference_annotation = Utils.get_annotation_with_source(ldm_attribute, "il_mapping")
+                                    if the_reference_annotation is None: 
+                                        the_reference_annotation = ELAnnotation()
+                                        the_reference_annotation_directive = Utils.get_annotation_directive(ldm_attribute.eContainer().eContainer(), "key")
+                                        the_reference_annotation.source = the_reference_annotation_directive
+                                        ldm_attribute.eAnnotations.append(the_reference_annotation)
+
+                                    details = the_reference_annotation.details
+
+                                    detail1 = ELStringToStringMapEntry()
+                                    detail1.key = "il_column"
+                                    detail1.value = relational_object_Name
+                                    details.append(detail1)
+
+
+
+    def get_ldm_attribute(self, context,entity_name,attribute_name):
+        for eClassifier in context.input_tables_package.eClassifiers:
+            if isinstance(eClassifier,ELClass):
+                for feature in eClassifier.eStructuralFeatures:
+                    if isinstance(feature,ELAttribute) and eClassifier.name == entity_name:
+                        if feature.name == attribute_name:
+                            return feature 
+                    if isinstance(feature,ELReference):
+                        if feature.name == attribute_name:
+                            return feature 
+
+    def get_ldm_entity(self, context,entity_name):
+        for eClassifier in context.input_tables_package.eClassifiers:
+            if isinstance(eClassifier,ELClass):
+                if eClassifier.name == entity_name:
+                    return eClassifier 
+