Small fixes in Extract-related Infrastructure (#3290)

Based on the logic that almost everyone who wants to extract data also wants to load it, we can filter out createable objects without naming them.

More careful filtering logic. Before, if filtering filtered out every object, the filter was interpreted as missing and ignored. 

Now it is checked as None when missing and empty list means empty.

Before, when synthesizing a declaration we didn't expand the field definitions from FIELDS(ALL) to the real field lisst.

Bugs in the testing infrastructure were fixed (e.g. confusing fake property declarations for objects and query responses that were not correlated with requests)

cumulusci/salesforce_api/filterable_objects.py

@@ -12,223 +12,42 @@
 
 NOT_EXTRACTABLE = NOT_COUNTABLE + (
     "%Share",
-    "AccountBrandShare",
-    "AccountShare",
-    "AccountUserTerritory2View",
-    "ActionPlanShare",
-    "ActionPlanTemplateShare",
-    "AgentWorkShare",
-    "AnalysisSetupStatusEvent",
-    "ApptBundleConfigShare",
-    "ApptBundlePolicyShare",
-    "AssetShare",
-    "AuthorizationFormConsentShare",
+    "%Access",
+    "%History",
+    "%Permission",
+    "%PermissionSet",
+    "%Permissions",
     "AuthorizationFormDataUse",
-    "AuthorizationFormDataUseHistory",
-    "AuthorizationFormDataUseShare",
-    "AuthorizationFormShare",
-    "BudgetShare",
-    "BusinessBrandShare",
-    "CalendarViewShare",
-    "CampaignShare",
-    "CaseShare",
-    "ChangeRequestShare",
-    "ChannelProgramLevelShare",
-    "ChannelProgramMemberShare",
-    "ChannelProgramShare",
-    "CommSubscriptionChannelTypeShare",
-    "CommSubscriptionConsentShare",
-    "CommSubscriptionShare",
-    "ContactPointAddressShare",
-    "ContactPointConsentShare",
-    "ContactPointEmailShare",
-    "ContactPointPhoneShare",
-    "ContactPointTypeConsentShare",
-    "ContactRequestShare",
-    "ContactShare",
-    "ContentUserSubscription",
-    "ContentWorkspacePermission",
     "CustomHelpMenuSection",
-    "CustomObjectUserLicenseMetrics",
-    "CustomPermission",
-    "CustomPermissionDependency",
-    "CustomerShare",
-    "DataIntegrationRecordPurchasePermission",
-    "DataPrepServiceLocatorShare",
     "DataUseLegalBasis",
-    "DataUseLegalBasisHistory",
-    "DataUseLegalBasisShare",
     "DataUsePurpose",
-    "DataUsePurposeHistory",
-    "DataUsePurposeShare",
-    "DatasetAccess",
-    "DelegatedAccountShare",
-    "DialerCallUsageShare",
-    "DocumentChecklistItemShare",
-    "EngagementChannelTypeShare",
-    "ExpenseReportShare",
-    "ExpenseShare",
     "ExternalDataUserAuth",
-    "ExternalEventMappingShare",
     "FieldPermissions",
-    "FlowInterviewLogShare",
-    "FlowInterviewShare",
-    "FlowTestResultShare",
-    "ForecastingShare",
-    "ForecastingUserPreference",
     "Group",
     "GroupMember",
-    "ImageShare",
-    "IncidentShare",
-    "IndividualShare",
-    "JobProfileShare",
-    "KnowledgeableUser",
-    "LeadShare",
-    "ListEmailShare",
-    "LiveAgentSessionShare",
-    "LiveChatObjectAccessConfig",
-    "LiveChatObjectAccessDefinition",
-    "LiveChatTranscriptShare",
-    "LiveChatUserConfig",
-    "LiveChatUserConfigProfile",
-    "LiveChatUserConfigUser",
-    "LocationShare",
-    "LocationTrustMeasureShare",
-    "MacroShare",
-    "MacroUsageShare",
-    "MaintenancePlanShare",
-    "MaintenanceWorkRuleShare",
-    "MessagingEndUser",
-    "MessagingEndUserHistory",
-    "MessagingEndUserShare",
-    "MessagingSessionShare",
-    "MutingPermissionSet",
-    "NetworkUserHistoryRecent",
-    "ObjectPermissions",
-    "OmniSupervisorConfigUser",
-    "OpportunityShare",
-    "OrderShare",
-    "OutgoingEmail",
-    "OutgoingEmailRelation",
-    "PartnerFundAllocationShare",
-    "PartnerFundClaimShare",
-    "PartnerFundRequestShare",
-    "PartnerMarketingBudgetShare",
-    "PartyConsentShare",
-    "PendingServiceRoutingShare",
     "PermissionSet",
     "PermissionSetAssignment",
     "PermissionSetGroup",
     "PermissionSetGroupComponent",
-    "PermissionSetLicense",
     "PermissionSetLicenseAssign",
     "PermissionSetTabSetting",
-    "PortalDelegablePermissionSet",
-    "PresenceUserConfig",
-    "PresenceUserConfigProfile",
-    "PresenceUserConfigUser",
-    "PricebookShare",
-    "ProblemShare",
-    "ProcessExceptionShare",
-    "ProductItemShare",
-    "ProductRequestShare",
-    "ProductServiceCampaignShare",
-    "ProductTransferShare",
-    "ProfileSkillShare",
-    "ProfileSkillUser",
-    "ProfileSkillUserFeed",
-    "ProfileSkillUserHistory",
-    "PromptActionShare",
-    "PromptErrorShare",
-    "QuickTextShare",
-    "QuickTextUsageShare",
-    "QuoteShare",
-    "RecordsetFilterCriteriaShare",
-    "RecurrenceScheduleShare",
-    "ReturnOrderShare",
-    "SOSSessionShare",
-    "SchedulingConstraintShare",
-    "ScorecardShare",
-    "SellerShare",
-    "ServiceAppointmentShare",
-    "ServiceContractShare",
-    "ServiceCrewShare",
-    "ServiceResourcePreferenceShare",
-    "ServiceResourceShare",
-    "ServiceTerritoryShare",
-    "SetupEntityAccess",
-    "SharingRecordCollectionShare",
-    "SharingUserCoverage",
-    "ShiftPatternShare",
-    "ShiftShare",
-    "ShiftTemplateShare",
-    "ShipmentShare",
-    "SkillUser",
-    "SocialPostShare",
-    "StreamingChannelShare",
-    "SurveyEngagementContextShare",
-    "SurveyInvitationShare",
-    "SurveyShare",
-    "TimeSheetShare",
-    "TodayGoalShare",
-    "TopicUserEvent",
-    "TranslationUser",
-    "TravelModeShare",
     "User",
-    "UserAccountTeamMember",
     "UserAppInfo",
     "UserAppMenuCustomization",
-    "UserAppMenuCustomizationShare",
-    "UserAppMenuItem",
-    "UserChangeEvent",
-    "UserConfigTransferButton",
-    "UserConfigTransferSkill",
     "UserCustomBadge",
     "UserCustomBadgeLocalization",
     "UserEmailPreferredPerson",
-    "UserEmailPreferredPersonShare",
-    "UserEntityAccess",
-    "UserFeed",
-    "UserFieldAccess",
-    "UserLicense",
     "UserListView",
     "UserListViewCriterion",
-    "UserLogin",
-    "UserNavItem",
     "UserPackageLicense",
-    "UserPermissionAccess",
     "UserPreference",
     "UserProvAccount",
     "UserProvAccountStaging",
     "UserProvMockTarget",
     "UserProvisioningConfig",
     "UserProvisioningLog",
     "UserProvisioningRequest",
-    "UserProvisioningRequestShare",
-    "UserRecordAccess",
     "UserRole",
-    "UserServicePresence",
-    "UserServicePresenceShare",
-    "UserSetupAppInfo",
-    "UserSetupEntityAccess",
-    "UserShare",
-    "UserTeamMember",
-    "UserTerritory2Association",
-    "VideoCallShare",
-    "VisualforceAccessMetrics",
-    "VoiceCallShare",
-    "WarrantyTermShare",
-    "WorkAccess",
-    "WorkAccessShare",
-    "WorkBadgeDefinitionShare",
-    "WorkOrderShare",
-    "WorkPlanSelectionRuleShare",
-    "WorkPlanShare",
-    "WorkPlanTemplateShare",
-    "WorkStepTemplateShare",
-    "WorkThanksShare",
-    "WorkTypeGroupShare",
-    "WorkTypeShare",
 )
 
 # Generated with these patterns:
@@ -240,8 +59,6 @@
 #     "NetworkUserHistoryRecent",
 #     "ObjectPermissions",
 #     "OmniSupervisorConfigUser",
-#     "OpportunityShare",
-#     "OrderShare",
 #     "OutgoingEmail",
 #     "OutgoingEmailRelation",
 # )
@@ -250,7 +67,7 @@
 #
 # patterns_to_ignore = NOT_EXTRACTABLE_WITHOUT_NOT_COUNTABLE
 
-# names = [obj["name"] for obj in sf.describe()["sobjects"]]
+# names = [obj["name"] for obj in sf.describe()["sobjects"] if obj["createable"]]
 
 # regexps_to_ignore = [
 #     re.compile(pat.replace("%", ".*"), re.IGNORECASE) for pat in patterns_to_ignore

cumulusci/salesforce_api/org_schema.py

@@ -105,6 +105,10 @@ class Filters(Enum):
     populated = SObject.count > 0  # does it have data in the org?
 
 
+# TODO: Profiling and optimizing of the
+#      SQL parts. After the object is frozen,
+#      all query-sets can be cached as
+#      dicts and lists
 class Schema:
     """Represents an org's schema, cached from describe() calls"""
 
@@ -122,7 +126,7 @@ def __init__(self, engine, schema_path, filters: T.Sequence[Filters] = ()):
     @property
     def sobjects(self):
         query = self.session.query(SObject)
-        if self.included_objects:
+        if self.included_objects is not None:
             query = query.filter(SObject.name.in_(self.included_objects))
         return query
 
@@ -133,7 +137,7 @@ def __getitem__(self, name):
             raise KeyError(f"No sobject named `{name}`")
 
     def __contains__(self, name):
-        return self.sobjects.filter_by(name=name).all()
+        return bool(self.sobjects.filter_by(name=name).first())
 
     def keys(self):
         return [x.name for x in self.sobjects.all()]
@@ -144,6 +148,9 @@ def values(self):
     def items(self):
         return [(obj.name, obj) for obj in self.sobjects]
 
+    def get(self, name: str):
+        return self.sobjects.filter_by(name=name).first()
+
     def block_writing(self):
         """After this method is called, the database can't be updated again"""
         # changes don't get saved back to the gzip
@@ -177,7 +184,9 @@ def __repr__(self):
 
     def add_counts(self, counts: T.Dict[str, int]):
         for objname, count in counts.items():
-            self[objname].count = count
+            obj = self.get(objname)
+            if obj:
+                obj.count = count
         self.includes_counts = True
 
     def populate_cache(
@@ -371,6 +380,7 @@ def get_org_schema(
         if Filters.extractable in filters:
             filters.add(Filters.queryable)
             filters.add(Filters.retrieveable)
+            filters.add(Filters.createable)  # so we can load again later
             patterns_to_ignore += NOT_EXTRACTABLE
 
         logger = logger or getLogger(__name__)
@@ -416,13 +426,13 @@ def get_org_schema(
 
             if Filters.populated in filters:
                 # another way to compute this might be by querying the first ID
-                objs_cached = [
+                objs_to_include = [
                     objname for objname, count in populated_objs.items() if count > 0
                 ]
             else:
-                objs_cached = [objname for objname, count in populated_objs.items()]
+                objs_to_include = [objname for objname, _ in populated_objs.items()]
 
-            schema.included_objects = objs_cached
+            schema.included_objects = objs_to_include
             schema.block_writing()
             # save a gzipped copy for later
             tempdb.zip_database(schema_path)

cumulusci/salesforce_api/org_schema_models.py

@@ -127,6 +127,10 @@ class SObject(OrgSchemaModelMixin, Base):
     actionOverrides = Column(SequenceType)
     count = Column(Integer)
 
+    @property
+    def extractable(self):
+        return self.createable and self.queryable and self.retrieveable
+
 
 field_references = Table(
     "references",

cumulusci/tasks/bulkdata/extract_dataset_utils/calculate_dependencies.py

@@ -77,14 +77,18 @@ def extend_declarations_to_include_referenced_tables(
         my_dependencies = dependencies.get(sf_object, ())
         for dep in my_dependencies:
             target_table = dep.table_name_to
-            if target_table not in decls and target_table not in NOT_EXTRACTABLE:
+            sobj = schema.get(target_table)
+            target_extractable = (
+                target_table not in NOT_EXTRACTABLE and sobj and sobj.extractable
+            )
+            if target_table not in decls and target_extractable:
                 required_fields = [
                     field.name
                     for field in schema[target_table].fields.values()
                     if field.requiredOnCreate
                 ]
                 decls[target_table] = synthesize_declaration_for_sobject(
-                    target_table, required_fields
+                    target_table, required_fields, schema[target_table].fields
                 )
 
                 new_dependencies = _calculate_dependencies_for_sobject(