Merge branch 'main' into bugfix/1gp-error

.github/workflows/feature_test.yml

@@ -42,6 +42,12 @@ jobs:
             matrix:
                 os: [macos-latest, sfdc-ubuntu-latest, sfdc-windows-latest]
                 python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+                exclude:
+                    - os: macos-latest
+                      python-version: 3.8
+                include:
+                    - os: macos-13
+                      python-version: 3.8
         steps:
             - uses: actions/checkout@v2
             - name: Set up Python

cumulusci/__about__.py

@@ -1 +1 @@
-__version__ = "3.84.2"
+__version__ = "3.86.1"

cumulusci/core/dependencies/dependencies.py

@@ -544,6 +544,7 @@ class UnmanagedDependency(StaticDependency, abc.ABC):
     subfolder: Optional[str] = None
     namespace_inject: Optional[str] = None
     namespace_strip: Optional[str] = None
+    collision_check: Optional[bool] = None
 
     def _get_unmanaged(self, org: OrgConfig):
         if self.unmanaged is None:

cumulusci/core/dependencies/tests/test_dependencies.py

@@ -903,3 +903,13 @@ def test_parse_unmanaged_dependency(self):
             }
         )
         assert isinstance(u, UnmanagedZipURLDependency)
+
+        u = parse_dependency(
+            {
+                "github": "https://github.com/Test/TestRepo",
+                "ref": "aaaaaaaa",
+                "collision_check": False,
+                "namespace_inject": "ns",
+            }
+        )
+        assert isinstance(u, UnmanagedGitHubRefDependency)

cumulusci/core/runtime.py

@@ -83,7 +83,8 @@ def _load_project_config(self, *args, **kwargs):
         self.project_config = self.project_config_cls(
             self.universal_config, *args, **kwargs
         )
-        self.project_config._add_tasks_directory_to_python_path()
+        if self.project_config is not None:
+            self.project_config._add_tasks_directory_to_python_path()
 
     def _load_keychain(self):
         if self.keychain is not None:

cumulusci/core/tests/test_datasets_e2e.py

@@ -129,6 +129,7 @@ def count(sobject):
         objs["Account"].remove("Description")
         objs["Account"].remove("History__c")
         objs["Account"].remove("ns__Description__c")
+        objs["Account"].remove("ns__LinkedAccount__c")
         objs["Account"].remove("Primary_Contact__c")
         dataset.update_schema_subset(objs)
         timer.checkpoint("Updated Subset")

cumulusci/cumulusci.yml

@@ -1480,7 +1480,7 @@ project:
         namespace:
         install_class:
         uninstall_class:
-        api_version: "59.0"
+        api_version: "60.0"
     git:
         default_branch: master
         prefix_feature: feature/

cumulusci/oauth/salesforce.py

@@ -21,6 +21,19 @@
 PROD_LOGIN_URL = os.environ.get("SF_PROD_LOGIN_URL") or "https://login.salesforce.com"
 
 
+def update_login_urls():
+    """
+    Updating to setup the environment variables dynamically
+    """
+    global PROD_LOGIN_URL, SANDBOX_LOGIN_URL
+    PROD_LOGIN_URL = (
+        os.environ.get("SF_PROD_LOGIN_URL") or "https://login.salesforce.com"
+    )
+    SANDBOX_LOGIN_URL = (
+        os.environ.get("SF_SANDBOX_LOGIN_URL") or "https://test.salesforce.com"
+    )
+
+
 def jwt_session(
     client_id, private_key, username, url=None, auth_url=None, is_sandbox=False
 ):
@@ -31,6 +44,7 @@ def jwt_session(
     :param username: Username to authenticate as
     :param url: Org's instance_url
     """
+    update_login_urls()
     if auth_url:
         aud = (
             SANDBOX_LOGIN_URL

cumulusci/oauth/tests/test_salesforce.py

@@ -1,3 +1,4 @@
+import os
 from unittest import mock
 
 import pytest
@@ -28,10 +29,12 @@ def test_jwt_session(encode):
 @mock.patch("cumulusci.oauth.salesforce.jwt.encode")
 def test_jwt_session__enhanced_domains_enabled(encode):
     # raise an assertion error if the registered url was not accessed
+    os.environ["SF_PROD_LOGIN_URL"] = "https://login.test1.salesforce.com"
+    os.environ["SF_SANDBOX_LOGIN_URL"] = "https://test.test1.salesforce.com"
     with responses.RequestsMock(assert_all_requests_are_fired=True) as rsps:
         rsps.add(
             responses.POST,
-            "https://test.salesforce.com/services/oauth2/token",
+            "https://test.test1.salesforce.com/services/oauth2/token",
             body='{"message":"well done mate!"}',
             status=200,
         )

cumulusci/tasks/bulkdata/load.py

@@ -835,7 +835,7 @@ def _generate_contact_id_map_for_person_accounts(
 
                 # Join maps together to get tuple (Contact ID, Contact SF ID) to insert into step's ID Table.
                 if self._old_format:
-                    yield (contact_mapping.table + "-" + contact_id, contact_sf_id)
+                    yield (contact_mapping.table + "-" + str(contact_id), contact_sf_id)
                 else:
                     yield (contact_id, contact_sf_id)
 

cumulusci/tasks/bulkdata/mapping_parser.py

@@ -612,13 +612,13 @@ def _infer_and_validate_lookups(mapping: Dict, sf: Salesforce):
             {f["name"]: f for f in getattr(sf, m.sf_object).describe()["fields"]}
         )
 
-        for lookup in m.lookups.values():
+        for lookup_name, lookup in m.lookups.items():
             if lookup.after:
                 # If configured by the user, skip.
                 # TODO: do we need more validation here?
                 continue
 
-            field_describe = describe.get(lookup.name, {})
+            field_describe = describe.get(lookup_name, {})
             reference_to_objects = field_describe.get("referenceTo", [])
             target_objects = []
 
@@ -634,7 +634,7 @@ def _infer_and_validate_lookups(mapping: Dict, sf: Salesforce):
                         target_objects.append(sf_object)
                     else:
                         logger.error(
-                            f"The lookup {sf_object} is not a valid lookup for {lookup.name} in sf_object: {m.sf_object}"
+                            f"The lookup {sf_object} is not a valid lookup for {lookup_name} in sf_object: {m.sf_object}"
                         )
                         fail = True
                 except KeyError:
@@ -660,7 +660,7 @@ def _infer_and_validate_lookups(mapping: Dict, sf: Salesforce):
                 ]
                 if not all([target_index < idx for target_index in target_indices]):
                     logger.error(
-                        f"All included target objects ({','.join(target_objects)}) for the field {m.sf_object}.{lookup.name} "
+                        f"All included target objects ({','.join(target_objects)}) for the field {m.sf_object}.{lookup_name} "
                         f"must precede {m.sf_object} in the mapping."
                     )
                     fail = True

cumulusci/tasks/bulkdata/query_transformers.py

@@ -1,7 +1,7 @@
 import typing as T
 from functools import cached_property
 
-from sqlalchemy import and_, func, text
+from sqlalchemy import String, and_, func, text
 from sqlalchemy.orm import Query, aliased
 
 from cumulusci.core.exceptions import BulkDataException
@@ -75,7 +75,7 @@ def join_for_lookup(lookup):
                 return (
                     lookup.aliased_table,
                     lookup.aliased_table.columns.id
-                    == str(lookup.table) + "-" + value_column,
+                    == str(lookup.table) + "-" + func.cast(value_column, String),
                 )
             else:
                 return (

cumulusci/tasks/bulkdata/tests/test_load.py

@@ -871,20 +871,22 @@ def test_get_statics_record_type_not_matched(self):
         assert "RecordType" in str(e.value)
 
     def test_query_db__joins_self_lookups(self):
+        """SQL file in Old Format"""
         _validate_query_for_mapping_step(
             sql_path=Path(__file__).parent / "test_query_db__joins_self_lookups.sql",
             mapping=Path(__file__).parent / "test_query_db__joins_self_lookups.yml",
             mapping_step_name="Update Accounts",
-            expected="""SELECT accounts.id AS accounts_id, accounts."Name" AS "accounts_Name", cumulusci_id_table_1.sf_id AS cumulusci_id_table_1_sf_id FROM accounts LEFT OUTER JOIN cumulusci_id_table AS cumulusci_id_table_1 ON cumulusci_id_table_1.id = ? || accounts.parent_id ORDER BY accounts.parent_id""",
+            expected="""SELECT accounts.id AS accounts_id, accounts."Name" AS "accounts_Name", cumulusci_id_table_1.sf_id AS cumulusci_id_table_1_sf_id FROM accounts LEFT OUTER JOIN cumulusci_id_table AS cumulusci_id_table_1 ON cumulusci_id_table_1.id = ? || cast(accounts.parent_id as varchar) ORDER BY accounts.parent_id""",
             old_format=True,
         )
 
     def test_query_db__joins_polymorphic_lookups(self):
+        """SQL File in New Format (Polymorphic)"""
         _validate_query_for_mapping_step(
             sql_path=Path(__file__).parent / "test_query_db_joins_lookups.sql",
             mapping=Path(__file__).parent / "test_query_db_joins_lookups.yml",
             mapping_step_name="Update Event",
-            expected="""SELECT events.id AS events_id, events."Subject" AS "events_Subject", cumulusci_id_table_1.sf_id AS cumulusci_id_table_1_sf_id FROM events LEFT OUTER JOIN cumulusci_id_table AS cumulusci_id_table_1 ON cumulusci_id_table_1.id = ? || events."WhoId" ORDER BY events."WhoId" """,
+            expected="""SELECT events.id AS events_id, events."Subject" AS "events_Subject", cumulusci_id_table_1.sf_id AS cumulusci_id_table_1_sf_id FROM events LEFT OUTER JOIN cumulusci_id_table AS cumulusci_id_table_1 ON cumulusci_id_table_1.id = ? || cast(events."WhoId" as varchar) ORDER BY events."WhoId" """,
         )
 
     @responses.activate

cumulusci/tasks/bulkdata/tests/test_mapping_parser.py

@@ -1071,6 +1071,39 @@ def test_validate_and_inject_mapping_injects_namespaces(self):
 
         assert list(ms.fields.keys()) == ["ns__Description__c"]
 
+    @responses.activate
+    def test_validate_and_inject_mapping_injects_namespaces__validates_lookup(self):
+        """Test to verify that with namespace inject, we validate lookups correctly"""
+        mock_describe_calls()
+        # Note: ns__Description__c is a mock field added to our stored, mock describes (in JSON)
+        mapping = parse_from_yaml(
+            StringIO(
+                """Insert Accounts:
+                  sf_object: Account
+                  table: Account
+                  fields:
+                    - Description__c
+                  lookups:
+                    LinkedAccount__c:
+                      table: Account"""
+            )
+        )
+        ms = mapping["Insert Accounts"]
+        org_config = DummyOrgConfig(
+            {"instance_url": "https://example.com", "access_token": "abc123"}, "test"
+        )
+
+        assert ms.validate_and_inject_namespace(
+            org_config.salesforce_client,
+            "ns",
+            DataOperationType.INSERT,
+            inject_namespaces=True,
+        )
+        # Here we verify that the field ns__LinkedAccount__c does lookup
+        # to sobject Account inside of describe
+        _infer_and_validate_lookups(mapping, org_config.salesforce_client)
+        assert list(ms.lookups.keys()) == ["ns__LinkedAccount__c"]
+
     @responses.activate
     def test_validate_and_inject_mapping_removes_namespaces(self):
         mock_describe_calls()

cumulusci/tasks/bulkdata/tests/test_query_db__joins_self_lookups.sql

@@ -1,6 +1,6 @@
 BEGIN TRANSACTION;
 CREATE TABLE "accounts" (
-	id VARCHAR(255) NOT NULL, 
+	id INTEGER NOT NULL, 
 	"Name" VARCHAR(255), 
 	"parent_id" VARCHAR(255), 
 	PRIMARY KEY (id)

cumulusci/tasks/robotframework/libdoc.py

@@ -123,7 +123,7 @@ def _run_task(self):
                     if library_name.endswith(".robot"):
                         libdoc = ResourceDocBuilder().build(library_name)
                     else:
-                        libdoc = DocumentationBuilder(library_name).build(library_name)
+                        libdoc = DocumentationBuilder().build(library_name)
                     kwfile.add_keywords(libdoc)
 
                 # if we get here, we were able to process the file correctly

cumulusci/utils/xml/robot_xml.py

@@ -1,7 +1,7 @@
 import re
 from typing import Callable, Dict, NamedTuple
 
-from robot.api import ExecutionResult, ResultVisitor
+from robot.api import ExecutionResult, ResultVisitor  # type: ignore
 from robot.result.model import TestCase
 
 UNITS = {
@@ -53,7 +53,8 @@ def log_perf_summary_from_xml(
 
     Supply a formatter that takes a PerfSummary triple if the default isn't a good fit:
 
-        f(test_name: str, metrics: Dict[str, float], test: robot.result.model.TestCase)"""
+        f(test_name: str, metrics: Dict[str, float], test: robot.result.model.TestCase)
+    """
     result = ExecutionResult(robot_xml)
     pl = _perf_logger(logger_func, formatter_func)
     next(pl)  # start the generator

docs/history.md

@@ -2,6 +2,52 @@
 
 <!-- latest-start -->
 
+## v3.86.1 (2024-05-06)
+
+<!-- Release notes generated using configuration in .github/release.yml at main -->
+
+**Full Changelog**: https://github.com/SFDO-Tooling/CumulusCI/compare/v3.86.0...v3.86.1
+
+<!-- latest-stop -->
+
+## v3.86.0 (2024-03-27)
+
+<!-- Release notes generated using configuration in .github/release.yml at main -->
+
+## What's Changed
+
+### Changes 🎉
+
+-   @W-15258380 - Reloading of login URL's from env before creating jwt session by [@lakshmi2506](https://github.com/lakshmi2506) in [#3765](https://github.com/SFDO-Tooling/CumulusCI/pull/3765)
+
+**Full Changelog**: https://github.com/SFDO-Tooling/CumulusCI/compare/v3.85.0...v3.86.0
+
+## v3.85.0 (2024-03-13)
+
+<!-- Release notes generated using configuration in .github/release.yml at main -->
+
+## What's Changed
+
+### Changes 🎉
+
+-   Fix Invalid Lookups of namespaced fields by [@aditya-balachander](https://github.com/aditya-balachander) in [#3759](https://github.com/SFDO-Tooling/CumulusCI/pull/3759)
+-   Fix Id Not Specified on Update Call by [@aditya-balachander](https://github.com/aditya-balachander) in [#3761](https://github.com/SFDO-Tooling/CumulusCI/pull/3761)
+-   Fix dependency parsing errors caused by `collision_check` option by [@jstvz](https://github.com/jstvz) in [#3760](https://github.com/SFDO-Tooling/CumulusCI/pull/3760)
+
+**Full Changelog**: https://github.com/SFDO-Tooling/CumulusCI/compare/v3.84.3...v3.85.0
+
+## v3.84.3 (2024-02-24)
+
+<!-- Release notes generated using configuration in .github/release.yml at main -->
+
+## What's Changed
+
+### Changes 🎉
+
+-   Fix: Type Error while loading Contacts when PersonAccount is enabled by [@jkasturi-sf](https://github.com/jkasturi-sf) in [#3756](https://github.com/SFDO-Tooling/CumulusCI/pull/3756)
+
+**Full Changelog**: https://github.com/SFDO-Tooling/CumulusCI/compare/v3.84.2...v3.84.3
+
 ## v3.84.2 (2024-02-20)
 
 <!-- Release notes generated using configuration in .github/release.yml at main -->
@@ -16,8 +62,6 @@
 
 **Full Changelog**: https://github.com/SFDO-Tooling/CumulusCI/compare/v3.84.1...v3.84.2
 
-<!-- latest-stop -->
-
 ## v3.84.1 (2024-02-01)
 
 <!-- Release notes generated using configuration in .github/release.yml at main -->

pyproject.toml

@@ -56,7 +56,7 @@ dependencies = [
     "selenium<4",
     "simple-salesforce==1.11.4",
     "snowfakery",
-    "SQLAlchemy",
+    "SQLAlchemy<2",
     "xmltodict",
 ]