Merge branch 'development' into fix/dev-2323-fix-fabs38

dataactcore/migrations/versions/0cf297fa927c_adding_unique_transaction_keys_to_.py

@@ -0,0 +1,42 @@
+"""Adding unique transaction keys to staging models
+
+Revision ID: 0cf297fa927c
+Revises: 94efce2e7882
+Create Date: 2019-03-21 17:14:34.938006
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '0cf297fa927c'
+down_revision = '94efce2e7882'
+branch_labels = None
+depends_on = None
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+def upgrade(engine_name):
+    globals()["upgrade_%s" % engine_name]()
+
+
+def downgrade(engine_name):
+    globals()["downgrade_%s" % engine_name]()
+
+
+
+
+
+def upgrade_data_broker():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('award_financial_assistance', sa.Column('afa_generated_unique', sa.Text(), nullable=True))
+    op.add_column('award_procurement', sa.Column('detached_award_proc_unique', sa.Text(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade_data_broker():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('award_procurement', 'detached_award_proc_unique')
+    op.drop_column('award_financial_assistance', 'afa_generated_unique')
+    # ### end Alembic commands ###
+

dataactcore/models/stagingModels.py

@@ -390,6 +390,7 @@ class AwardFinancialAssistance(Base):
     __tablename__ = "award_financial_assistance"
 
     award_financial_assistance_id = Column(Integer, primary_key=True)
+    afa_generated_unique = Column(Text)
     submission_id = Column(Integer,
                            ForeignKey("submission.submission_id", ondelete="CASCADE",
                                       name="fk_award_financial_assistance_submission_id"),
@@ -480,6 +481,7 @@ class AwardProcurement(Base):
     """Model for D1-Award (Procurement)."""
     __tablename__ = "award_procurement"
     award_procurement_id = Column(Integer, primary_key=True)
+    detached_award_proc_unique = Column(Text)
     submission_id = Column(Integer,
                            ForeignKey("submission.submission_id", ondelete="CASCADE",
                                       name="fk_award_procurement_submission_id"),

dataactcore/scripts/pull_fpds_data.py

@@ -725,12 +725,10 @@ def vendor_site_details_values(data, obj):
         obj['legal_entity_country_name'] = None
 
     # vendorOrganizationFactors sub-level
-    value_map = {'annualRevenue': 'annual_revenue',
-                 'isForeignOwnedAndLocated': 'foreign_owned_and_located',
+    value_map = {'isForeignOwnedAndLocated': 'foreign_owned_and_located',
                  'isLimitedLiabilityCorporation': 'limited_liability_corporat',
                  'isShelteredWorkshop': 'the_ability_one_program',
                  'isSubchapterSCorporation': 'subchapter_s_corporation',
-                 'numberOfEmployees': 'number_of_employees',
                  'organizationalType': 'organizational_type'}
 
     for key, value in value_map.items():

dataactcore/utils/fileD1.py

@@ -6,6 +6,7 @@
 file_model = DetachedAwardProcurement
 
 mapping = OrderedDict([
+    ('ContractTransactionUniqueKey', 'detached_award_proc_unique'),
     ('PIID', 'piid'),
     ('AwardModificationAmendmentNumber', 'award_modification_amendme'),
     ('Transaction Number', 'transaction_number'),
@@ -247,8 +248,6 @@
     ('Nonprofit Organization', 'nonprofit_organization'),
     ('Other Not For Profit Organization', 'other_not_for_profit_organ'),
     ('The AbilityOne Program', 'the_ability_one_program'),
-    ('NumberOfEmployees', 'number_of_employees'),
-    ('AnnualRevenue', 'annual_revenue'),
     ('Private University or College', 'private_university_or_coll'),
     ('State Controlled Institution of Higher Learning', 'state_controlled_instituti'),
     ('1862 Land Grant College', 'c1862_land_grant_college'),
@@ -300,6 +299,7 @@ def query_data(session, agency_code, agency_type, start, end):
 
 def initial_query(session):
     return session.query(*[
+        file_model.detached_award_proc_unique,
         file_model.piid,
         file_model.award_modification_amendme,
         file_model.transaction_number,
@@ -541,8 +541,6 @@ def initial_query(session):
         file_model.nonprofit_organization,
         file_model.other_not_for_profit_organ,
         file_model.the_ability_one_program,
-        file_model.number_of_employees,
-        file_model.annual_revenue,
         file_model.private_university_or_coll,
         file_model.state_controlled_instituti,
         file_model.c1862_land_grant_college,

dataactcore/utils/fileD2.py

@@ -6,6 +6,7 @@
 file_model = PublishedAwardFinancialAssistance
 
 mapping = OrderedDict([
+    ('AssistanceTransactionUniqueKey', 'afa_generated_unique'),
     ('FAIN', 'fain'),
     ('AwardModificationAmendmentNumber', 'award_modification_amendme'),
     ('URI', 'uri'),
@@ -130,6 +131,7 @@ def initial_query(session):
             The base query (a select from the PublishedAwardFinancialAssistance table with the specified columns).
     """
     return session.query(
+        file_model.afa_generated_unique,
         file_model.fain,
         file_model.award_modification_amendme,
         file_model.uri,

dataactvalidator/config/awardFields.csv

@@ -1,4 +1,5 @@
 fieldname,fieldname_short,required,data_type,field_length,padded_flag
+assistancetransactionuniquekey,afa_generated_unique,FALSE,str,,
 actiondate,action_date,TRUE,str,8,
 actiontype,action_type,FALSE,str,1,
 actiontypedescriptiontag,action_type_description,FALSE,str,,

dataactvalidator/config/awardFieldsRaw.csv

@@ -1,4 +1,5 @@
 fieldname,fieldname_short,required,data_type,field_length
+AssistanceTransactionUniqueKey,afa_generated_unique,ASP: Optional,string,
 ActionDate,action_date,ASP: Required,string,8
 ActionType,action_type,ASP: Conditional per Validation Rule,string,1
 ActionTypeDescriptionTag,action_type_description,ASP: Optional,string,

dataactvalidator/config/awardProcurementFields.csv

@@ -1,4 +1,5 @@
 fieldname,fieldname_short,required,data_type,field_length,padded_flag
+contracttransactionuniquekey,detached_award_proc_unique,FALSE,str,,
 piid,piid,FALSE,str,,
 awardingsubtieragencycode,awarding_sub_tier_agency_c,FALSE,str,,
 awardingsubtieragencyname,awarding_sub_tier_agency_n,FALSE,str,,
@@ -258,6 +259,4 @@ cage_code,cage_code,FALSE,str,,
 inherently_governmental_functions,inherently_government_func,FALSE,str,,
 inherently_governmental_functions_description_tag,inherently_government_desc,FALSE,str,,
 organizationaltype,organizational_type,FALSE,str,,
-numberofemployees,number_of_employees,FALSE,str,,
-annualrevenue,annual_revenue,FALSE,str,,
 totaldollarsobligated,total_obligated_amount,FALSE,str,,

dataactvalidator/config/awardProcurementFieldsRaw.csv

@@ -1,4 +1,5 @@
 fieldname,fieldname_short,required,data_type,field_length
+ContractTransactionUniqueKey,detached_award_proc_unique,optional,string,
 PIID,piid,optional,string,
 AwardingSubTierAgencyCode,awarding_sub_tier_agency_c,optional,string,
 AwardingSubTierAgencyName,awarding_sub_tier_agency_n,optional,string,
@@ -258,6 +259,4 @@ CAGE Code,cage_code,optional,string,
 Inherently Governmental Functions,inherently_government_func,optional,string,
 Inherently Governmental Functions Description Tag,inherently_government_desc,optional,string,
 OrganizationalType,organizational_type,optional,string,
-NumberOfEmployees,number_of_employees,optional,string,
-AnnualRevenue,annual_revenue,optional,string,
 TotalDollarsObligated,total_obligated_amount,optional,string,

tests/unit/dataactcore/factories/staging.py

@@ -155,6 +155,7 @@ class Meta:
         model = stagingModels.AwardFinancialAssistance
 
     award_financial_assistance_id = None
+    afa_generated_unique = fuzzy.FuzzyText()
     submission_id = fuzzy.FuzzyInteger(9999)
     job_id = fuzzy.FuzzyInteger(9999)
     row_number = fuzzy.FuzzyInteger(9999)
@@ -365,6 +366,7 @@ class Meta:
         model = stagingModels.AwardProcurement
 
     award_procurement_id = None
+    detached_award_proc_unique = fuzzy.FuzzyText()
     submission_id = fuzzy.FuzzyInteger(9999)
     job_id = fuzzy.FuzzyInteger(9999)
     row_number = fuzzy.FuzzyInteger(1, 9999)