Update Generation rules creator for Asset Encumberance #1342

#1342

bird/birdseed_creator/src/context/context.py

@@ -29,10 +29,8 @@ class Context(object):
     persist_to_regdna = True
     inScopeFileDirectory = ""
     add_pks_to_input_layer_from_website = True 
-    reporting_framework = "FINREP"
-    reporting_framework_version = ["3","3.0-Ind","FINREP 3.0-Ind"]
     match_domains_in_generation_file = False
-    input_layer_name = "Input Layer 6.2"
+    input_layer_name = "Input Layer 6.3"
     # the directory where we get our input files
     file_directory = ""
     # the directory where we save our outputs.

bird/birdseed_creator/src/entry_points/create_outline_transformations_ae.py

@@ -0,0 +1,59 @@
+# coding=UTF-8#
+# Copyright (c) 2020 Bird Software Solutions Ltd
+# This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License 2.0
+# which accompanies this distribution, and is available at
+# https://www.eclipse.org/legal/epl-2.0/
+#
+# SPDX-License-Identifier: EPL-2.0
+#
+# Contributors:
+#    Neil Mackenzie - initial API and implementation
+#
+from context.context import Context
+from context.sdd_context import SDDContext
+from entry_points.website_to_sddmodel import RunWebsiteToSDDModel
+from entry_points.sddmodel_to_datamodel_ae import RunSDDModelToDataModelAE 
+from persister.persist_to_file import PersistToFile
+from process_steps.sddmodel_plus_datamodel_to_outline_transformations.main_catagory_finder import MainCatagoryFinder
+from process_steps.sddmodel_plus_datamodel_to_outline_transformations.generation_rule_creator import GenerationRuleCreator
+
+
+class GenerateGenerationRulesAE:
+
+    def run(self,context,sdd_context):
+
+
+        # we will create an ecore4reg representation of BIRD also
+        # use codes not long names for columns
+        context.use_codes = True
+        # when finding  source columns form the input layer, related
+        # to items in the output layer we find columns with matching
+        # variables not matching domains
+        context.find_variable_with_same_domain = False
+        # find the main catagories related with reports.
+        MainCatagoryFinder().create_report_to_main_catogory_maps(context,sdd_context)
+        # create the generation rules
+        GenerationRuleCreator().generate_generation_rules(context,sdd_context)
+        # save the generated  files in useful formats
+
+
+
+
+if __name__ == '__main__':
+    sdd_context = SDDContext()
+    context = Context()
+    context.file_directory = '/workspaces/efbt/bird/birdseed_creator/resources'
+    context.output_directory = '/workspaces/efbt/bird/birdseed_creator/results' 
+    sdd_context.file_directory = '/workspaces/efbt/bird/birdseed_creator/resources'
+    sdd_context.output_directory = '/workspaces/efbt/bird/birdseed_creator/results'   
+
+
+    RunWebsiteToSDDModel().run(sdd_context)
+    RunSDDModelToDataModelAE().run(context,sdd_context)
+    GenerateGenerationRulesAE().run(context,sdd_context)
+    persister = PersistToFile()
+    persister.save_model_as_regdna_file(context)
+    persister.persist_generation_transformations(context)
+    persister.persist_generation_transformations_to_csv(context)
+

bird/birdseed_creator/src/entry_points/create_outline_transformations.py → bird/birdseed_creator/src/entry_points/create_outline_transformations_finrep.py

@@ -13,18 +13,17 @@
 from context.context import Context
 from context.sdd_context import SDDContext
 from entry_points.website_to_sddmodel import RunWebsiteToSDDModel
-from entry_points.sddmodel_to_datamodel import RunSDDModelToDataModel 
+from entry_points.sddmodel_to_datamodel_finrep import RunSDDModelToDataModelFinrep 
 from persister.persist_to_file import PersistToFile
 from process_steps.sddmodel_plus_datamodel_to_outline_transformations.main_catagory_finder import MainCatagoryFinder
 from process_steps.sddmodel_plus_datamodel_to_outline_transformations.generation_rule_creator import GenerationRuleCreator
-from utils.relationship_enricher import RelationshipEnricher
 
-class GenerateGenerationRules:
+
+class GenerateGenerationRulesFinrep:
 
     def run(self,context,sdd_context):
 
-        context.reporting_framework = "FINREP"
-        context.reporting_framework_version = ["3","3.0-Ind","FINREP 3.0-Ind"]
+
         # we will create an ecore4reg representation of BIRD also
         # use codes not long names for columns
         context.use_codes = True
@@ -49,8 +48,8 @@ def run(self,context,sdd_context):
     sdd_context.file_directory = '/workspaces/efbt/bird/birdseed_creator/resources'
     sdd_context.output_directory = '/workspaces/efbt/bird/birdseed_creator/results'    
     RunWebsiteToSDDModel().run(sdd_context)
-    RunSDDModelToDataModel().run(context,sdd_context)
-    GenerateGenerationRules().run(context,sdd_context)
+    RunSDDModelToDataModelFinrep().run(context,sdd_context)
+    GenerateGenerationRulesFinrep().run(context,sdd_context)
     persister = PersistToFile()
     persister.save_model_as_regdna_file(context)
     persister.persist_generation_transformations(context)

bird/birdseed_creator/src/entry_points/sddmodel_to_datamodel.py → bird/birdseed_creator/src/entry_points/sddmodel_to_datamodel_ae.py

@@ -17,21 +17,23 @@
 from entry_points.website_to_sddmodel import RunWebsiteToSDDModel 
 from context.context import Context
 from context.sdd_context import SDDContext
-from persister.persist_sdd import PersistSDD
 from process_steps.sddmodel_to_datamodel.translate_sddmodel_to_datamodel import TranslateSDDModelToDataModel
 from persister.persist_to_file import PersistToFile
 from utils.relationship_enricher import RelationshipEnricher
 
 
-class RunSDDModelToDataModel:
+class RunSDDModelToDataModelAE:
 
     def run(self,context,sdd_context):
         context.persist_to_regdna = True
         context.load_eil_from_website  = True
+        context.reporting_framework = "AE"
+        context.reporting_framework_version = ["AE 3.2"]
         TranslateSDDModelToDataModel().do_import(context,sdd_context)
         RelationshipEnricher().enrich(context)
 
 
+
 if __name__ == '__main__':
     sdd_context = SDDContext()
     context = Context()
@@ -40,10 +42,15 @@ def run(self,context,sdd_context):
     sdd_context.file_directory = '/workspaces/efbt/bird/birdseed_creator/resources'
     sdd_context.output_directory = '/workspaces/efbt/bird/birdseed_creator/results'    
     RunWebsiteToSDDModel().run(sdd_context)
-    RunSDDModelToDataModel().run(context,sdd_context)
+    RunSDDModelToDataModelAE().run(context,sdd_context)
     persister = PersistToFile()
     persister.save_model_as_regdna_file(context)
     persister.save_model_as_xmi_file(context)
+
+
+
+
+
 
 
 

bird/birdseed_creator/src/entry_points/sddmodel_to_datamodel_finrep.py

@@ -0,0 +1,53 @@
+# coding=UTF-8#
+# Copyright (c) 2020 Bird Software Solutions Ltd
+# This program and the accompanying materials
+# are made available under the terms of the Eclipse Public License 2.0
+# which accompanies this distribution, and is available at
+# https://www.eclipse.org/legal/epl-2.0/
+#
+# SPDX-License-Identifier: EPL-2.0
+#
+# Contributors:
+#    Neil Mackenzie - initial API and implementation
+#
+'''
+Created on 25 April 2022
+'''
+
+from entry_points.website_to_sddmodel import RunWebsiteToSDDModel 
+from context.context import Context
+from context.sdd_context import SDDContext
+from process_steps.sddmodel_to_datamodel.translate_sddmodel_to_datamodel import TranslateSDDModelToDataModel
+from persister.persist_to_file import PersistToFile
+from utils.relationship_enricher import RelationshipEnricher
+
+class RunSDDModelToDataModelFinrep:
+
+    def run(self,context,sdd_context):
+        context.persist_to_regdna = True
+        context.load_eil_from_website  = True
+        context.reporting_framework = "FINREP"
+        context.reporting_framework_version = ["3","3.0-Ind","FINREP 3.0-Ind"]
+        TranslateSDDModelToDataModel().do_import(context,sdd_context)
+        RelationshipEnricher().enrich(context)
+
+
+
+if __name__ == '__main__':
+    sdd_context = SDDContext()
+    context = Context()
+    context.file_directory = '/workspaces/efbt/bird/birdseed_creator/resources'
+    context.output_directory = '/workspaces/efbt/bird/birdseed_creator/results' 
+    sdd_context.file_directory = '/workspaces/efbt/bird/birdseed_creator/resources'
+    sdd_context.output_directory = '/workspaces/efbt/bird/birdseed_creator/results'    
+    RunWebsiteToSDDModel().run(sdd_context)
+    RunSDDModelToDataModelFinrep().run(context,sdd_context)
+    persister = PersistToFile()
+    persister.save_model_as_regdna_file(context)
+    persister.save_model_as_xmi_file(context)
+
+
+
+
+
+

bird/birdseed_creator/src/entry_points/website_to_sddmodel.py

@@ -23,10 +23,12 @@ def run(self,sdd_context):
 
 if __name__ == '__main__':
     sdd_context = SDDContext()
-    sdd_context.file_directory = 'C:\\Users\\LENOVO\\git\\efbt_regpot_desktop\\bird\\birdseed_creator\\resources'
-    sdd_context.output_directory = 'C:\\Users\\LENOVO\\git\\efbt_regpot_desktop\\bird\\birdseed_creator\\results'      
+    sdd_context.file_directory = '/workspaces/efbt/bird/birdseed_creator/resources'
+    sdd_context.output_directory = '/workspaces/efbt/bird/birdseed_creator/results'   
     RunWebsiteToSDDModel().run(sdd_context)
     persister =PersistSDD()
     persister.save_analysis_model_as_xmi_files(sdd_context)
-    persister.save_analysis_model_as_csv(sdd_context)
+    # persister.save_analysis_model_as_csv(sdd_context)
+
+
 

bird/birdseed_creator/src/persister/persist_to_file.py

@@ -12,7 +12,6 @@
 #
 import os
 from pyecore.resources import ResourceSet, URI
-from pyecore.resources.json import JsonResource
 from utils.utils import Utils
 
 
@@ -274,31 +273,33 @@ def persist_generation_transformations_to_csv(self, context):
                 for layer in rules_for_report.rulesForTable:
                     if not(layer.inputLayerTable is None):
                         table = layer.inputLayerTable.name
-                        for table_part in layer.rulesForTablePart:
-                            main_catagory = table_part.main_catagory
-                            main_catagory_name = context.main_catogory_to_name_map[main_catagory]
-
-                            table_and_part = table_part.table_and_part_tuple
-                            report_to_table_parts_file.write(amended_template_name + "," + table_part.name + ",\n")
-                            filter = context.table_parts_to_to_filter_map[table_and_part]
-                            for column in table_part.columns:
-                                if isinstance(column, SelectColumnAttributeAs) and not(column.attribute is None):
-                                    entity  = column.attribute.eContainer().name
-                                    attribute = column.attribute.name
-                                    lineage_type = "attribute"
-                                    missing = "Not Missing"
-                                else:  
-                                    entity  = ""
-                                    attribute = ""
-                                    missing = "Missing"
-                                    lineage_type = "tbd"
-
-                                variable_id = column.asAttribute.name
-
-
-                                f.write(amended_template_name +"," + table_part.name +","  +table+"," + filter + ","  +lineage_type+"," +entity+"," +attribute+"," +missing+",,,,,," +variable_id + ",\n")
                     else:
                         print ("no input layer table for " + rules_for_report.outputLayerCube.name  )
+                        table = "Null"
+                    for table_part in layer.rulesForTablePart:
+                        main_catagory = table_part.main_catagory
+                        main_catagory_name = context.main_catogory_to_name_map[main_catagory]
+
+                        table_and_part = table_part.table_and_part_tuple
+                        report_to_table_parts_file.write(amended_template_name + "," + table_part.name + ",\n")
+                        filter = context.table_parts_to_to_filter_map[table_and_part]
+                        for column in table_part.columns:
+                            if isinstance(column, SelectColumnAttributeAs) and not(column.attribute is None):
+                                entity  = column.attribute.eContainer().name
+                                attribute = column.attribute.name
+                                lineage_type = "attribute"
+                                missing = "Not Missing"
+                            else:  
+                                entity  = ""
+                                attribute = ""
+                                missing = "Missing"
+                                lineage_type = "tbd"
+
+                            variable_id = column.asAttribute.name
+
+
+                            f.write(amended_template_name +"," + table_part.name +","  +table+"," + filter + ","  +lineage_type+"," +entity+"," +attribute+"," +missing+",,,,,," +variable_id + ",\n")
+
                 f.close()
         report_to_table_parts_file.close
 

bird/birdseed_creator/src/process_steps/sddmodel_plus_datamodel_to_outline_transformations/main_catagory_finder.py

@@ -93,7 +93,7 @@ def create_report_to_main_catogory_map(self, context,sdd_context):
                         # set that we are interested in.
                         if version in context.reporting_framework_version:
 
-                            if (valid_to == "12/31/9999") or (valid_to == "31/12/9999"):
+                            if (valid_to == "12/31/9999") or (valid_to == "31/12/9999") or (valid_to == "9999-12-31"):
                                 valid_eba_tables.append(table_id)
         file_location = context.file_directory + os.sep + "axis_ordinate.csv"
         header_skipped = False
@@ -126,14 +126,17 @@ def create_report_to_main_catogory_map(self, context,sdd_context):
                     header_skipped = True
                 else:
                     axis_ordinate_id = row[0]
-                    if axis_ordinate_id.startswith("EBA_" + context.reporting_framework + "_EBA_"):
+                    axis_ordinate_id_string_prefix = "EBA_" + context.reporting_framework + "_EBA_"
+                    axis_ordinate_id_string_prefix_short = "EBA_" + context.reporting_framework 
+                    if axis_ordinate_id.startswith(axis_ordinate_id_string_prefix):
                         if axis_ordinate_id in valid_eba_axis_ordinates:
                             variable_id = row[1]
                             if variable_id == "EBA_MCY":
                                 # we find the report name by looking for the second
-                                # instance of the string FINREP_
-                                report_name = axis_ordinate_id[15:axis_ordinate_id.\
-                                              index("_" + context.reporting_framework,10)]
+                                # instance of the string FINREP_ or AE_ 
+                                axis_ordinate_id_string_prefix = "EBA_" + context.reporting_framework + "_EBA_"
+                                report_name = axis_ordinate_id[len(axis_ordinate_id_string_prefix):axis_ordinate_id.\
+                                              index("_" + context.reporting_framework,len(axis_ordinate_id_string_prefix_short))]
                                 member_id = row[2]
                                 amemnded_report_name = Utils.make_valid_id(report_name)
                                 if not(member_id in context.main_catagories_in_scope):