Exception: Failed to acquire started lock
self = <tests.unit.builtin_monitors.container_checker_test.ContainerCheckerMultiAccountTest testMethod=test_check_containers_with_api_key_annotations>

    def test_check_containers_with_api_key_annotations(self):
        WORKLOAD_POD_1_CONTAINER_3 = "workload-pod-1-container-3"
        WORKLOAD_POD_1_CONTAINER_2 = "workload-pod-1-container-2"
        WORKLOAD_POD_1_CONTAINER_1 = "workload-pod-1-container-1"
        WORKLOAD_POD_2_CONTAINER_1 = "workload-pod-2-container-1"
        WORKLOAD_POD_3_CONTAINER_1 = "workload-pod-3-container-1"
    
        k8s_namespace_1 = MockNamespace(
            name="namespace-1",
            annotations=process_annotations(
                {"log.config.scalyr.com/teams.1.secret": "scalyr-api-key-team-2"}
            ),
        )
    
        k8s_pod_1_namespace_1 = MockPod(
            namespace=k8s_namespace_1.name,
            name="k8s_pod_1_namespace_1",
            node_name="test_node",
            uid="lkjsefr",
            annotations=process_annotations(
                {
                    "log.config.scalyr.com/teams.1.secret": "scalyr-api-key-team-3",
                    "log.config.scalyr.com/teams.5.secret": "scalyr-api-key-team-4",
                    "log.config.scalyr.com/workload-pod-1-container-1.teams.1.secret": "scalyr-api-key-team-5",
                    "log.config.scalyr.com/workload-pod-1-container-2.teams.1.secret": "scalyr-api-key-team-6",
                    "log.config.scalyr.com/workload-pod-1-container-2.teams.2.secret": "scalyr-api-key-team-7",
                }
            ),
            container_names=[
                WORKLOAD_POD_1_CONTAINER_1,
                WORKLOAD_POD_1_CONTAINER_2,
                WORKLOAD_POD_1_CONTAINER_3,
            ],
            controller=Controller(
                name="controller_name_k8s_pod_1_namespace_1",
                kind="controller_kind_k8s_pod_1_namespace_1",
                labels={"controller_labels_k8s_pod_1_namespace_1": "value"},
            ),
        )
    
        k8s_pod_2_namespace_1 = MockPod(
            namespace=k8s_namespace_1.name,
            name="k8s_pod_2_namespace_1",
            node_name="test_node",
            uid="opimon",
            annotations=process_annotations({}),
            container_names=[
                WORKLOAD_POD_2_CONTAINER_1,
            ],
            controller=Controller(
                name="controller_name_k8s_pod_2_namespace_1",
                kind="controller_kind_k8s_pod_2_namespace_1",
                labels={"controller_labels_k8s_pod_2_namespace_1": "value"},
            ),
        )
    
        k8s_namespace_2 = MockNamespace(
            name="namespace-2", annotations=process_annotations({})
        )
        k8s_pod_3_namespace_2 = MockPod(
            namespace=k8s_namespace_2.name,
            name="k8s_pod_3_namespace_2",
            node_name="test_node",
            uid="fsdjlkh",
            annotations=process_annotations({}),
            container_names=[WORKLOAD_POD_3_CONTAINER_1],
            controller=Controller(
                name="controller_name_k8s_pod_3_namespace_2",
                kind="controller_kind_k8s_pod_3_namespace_2",
                labels={"controller_labels_k8s_pod_3_namespace_2": "value"},
            ),
        )
    
        get_containers_value = self._set_mocked_k8s_objects(
            [k8s_pod_1_namespace_1, k8s_pod_2_namespace_1, k8s_pod_3_namespace_2],
            [k8s_namespace_1, k8s_namespace_2],
        )
    
        run_state, exception_holder = self._start_check_containers_thread()
    
        # Wait for ContainerChecker to complete at least one loop
        run_state.wait_for_next_loop_start()
        run_state.wait_for_next_loop_start()
    
        assert len(exception_holder) == 0, "Expected no exceptions"
        assert len(self.mock_logger.exception.mock_calls) == 0, (
            "Some exceptions were logged: %s" % self.mock_logger.exception.mock_calls
        )
        assert len(self.mock_logger.warn.mock_calls) == 0, (
            "Some warnings were logged: %s" % self.mock_logger.warn.mock_calls
        )
        assert len(self.mock_logger.error.mock_calls) == 0, (
            "Some errors were logged: %s" % self.mock_logger.error.mock_calls
        )
    
        def expected_log_config(
            pod: MockPod, container_name: str, api_key: str, teams: List[str]
        ):
            log_config = {
                "parser": "docker",
                "path": f"/var/log/scalyr-agent2/containers/{container_name}.log",
                "parse_format": "test_parse_format",
                "rename_logfile": f"/unknown/{container_name}.log",
                "rename_no_original": None,
                "attributes": JsonObject(
                    {
                        "container_name": container_name,
                        "monitor": "agentKubernetes",
                        "container_id": container_name,
                        "pod_name": pod.name,
                        "pod_namespace": pod.namespace,
                        "_k8s_cn": "the-cluster-name",
                        "scalyr-category": "log",
                        "pod_uid": pod.uid,
                        "k8s_node": "test_node",
                        "_k8s_dn": pod.controller.name,
                        "_k8s_dl": self._flatten_labels(pod.controller.labels),
                        "_k8s_ck": pod.controller.kind,
                    }
                ),
                "k8s_pod_glob": "*",
                "k8s_namespace_glob": "*",
                "k8s_container_glob": "*",
                "lineGroupers": JsonArray(),
                "sampling_rules": JsonArray(),
                "redaction_rules": JsonArray(),
            }
            if teams:
                log_config["teams"] = JsonArray(
                    *[JsonObject({"secret": team}) for team in teams]
                )
            if api_key:
                log_config["api_key"] = api_key
    
            return log_config
    
        assert self.log_watcher.add_log_config.call_args_list == [
            call(
                "unittest_module",
                expected_log_config(
                    k8s_pod_1_namespace_1,
                    WORKLOAD_POD_1_CONTAINER_1,
                    "scalyr-api-key-team-5-value",
                    ["scalyr-api-key-team-5"],
                ),
                force_add=True,
            ),
            call(
                "unittest_module",
                expected_log_config(
                    k8s_pod_1_namespace_1,
                    WORKLOAD_POD_1_CONTAINER_2,
                    "scalyr-api-key-team-6-value",
                    ["scalyr-api-key-team-6", "scalyr-api-key-team-7"],
                ),
                force_add=True,
            ),
            call(
                "unittest_module",
                expected_log_config(
                    k8s_pod_1_namespace_1,
                    WORKLOAD_POD_1_CONTAINER_2,
                    "scalyr-api-key-team-7-value",
                    ["scalyr-api-key-team-6", "scalyr-api-key-team-7"],
                ),
                force_add=True,
            ),
            call(
                "unittest_module",
                expected_log_config(
                    k8s_pod_1_namespace_1,
                    WORKLOAD_POD_1_CONTAINER_3,
                    "scalyr-api-key-team-3-value",
                    ["scalyr-api-key-team-3", "scalyr-api-key-team-4"],
                ),
                force_add=True,
            ),
            call(
                "unittest_module",
                expected_log_config(
                    k8s_pod_1_namespace_1,
                    WORKLOAD_POD_1_CONTAINER_3,
                    "scalyr-api-key-team-4-value",
                    ["scalyr-api-key-team-3", "scalyr-api-key-team-4"],
                ),
                force_add=True,
            ),
            call(
                "unittest_module",
                expected_log_config(
                    k8s_pod_2_namespace_1,
                    WORKLOAD_POD_2_CONTAINER_1,
                    "scalyr-api-key-team-2-value",
                    None,
                ),
                force_add=True,
            ),
            call(
                "unittest_module",
                expected_log_config(
                    k8s_pod_3_namespace_2, WORKLOAD_POD_3_CONTAINER_1, None, None
                ),
                force_add=True,
            ),
        ]
    
        self.log_watcher.add_log_config.reset_mock()
    
        # Wait for the loop to complete and pause it to change pod annotations.
        with run_state.running_lock as lock:
>           run_state.wait_for_next_loop_start()

#x1B[1m#x1B[31mtests/unit/builtin_monitors/container_checker_test.py#x1B[0m:549: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <tests.unit.builtin_monitors.container_checker_test.MockRunState object at 0x10544aad0>

    def wait_for_next_loop_start(self):
        with self.next_loop_condition:
            if not self.next_loop_condition.wait(timeout=15):
>               raise Exception("Failed to acquire started lock")
#x1B[1m#x1B[31mE               Exception: Failed to acquire started lock#x1B[0m

#x1B[1m#x1B[31mtests/unit/builtin_monitors/container_checker_test.py#x1B[0m:41: Exception

subprocess.CalledProcessError: Command '['/Users/runner/work/scalyr-agent-2/scalyr-agent-2/.tox/py310-end2end/bin/python', '/Users/runner/work/scalyr-agent-2/scalyr-agent-2/scalyr_agent/agent_main.py', 'start']' returned non-zero exit status 1.
scalyr_api_key = '', scalyr_server = 'agent.scalyr.com'
scalyr_api_read_key = ''
server_host = 'test_basic-3.10-macos-11-9180500811-1-1716320620'
agent_paths = AgentPaths(configs_dir=PosixPath('/Users/runner/scalyr-agent-dev/config'), logs_dir=PosixPath('/Users/runner/scalyr-ag...nner/scalyr-agent-dev/log/agent.log'), agent_config_path=PosixPath('/Users/runner/scalyr-agent-dev/config/agent.json'))
agent_commander = <tests.end_to_end_tests.tools.AgentCommander object at 0x10cf52350>
default_config = {'api_key': '', 'logs': [{'attributes': {'parser': 'json'}, 'path': '/Users/runner/scalyr-agent-dev/log/test.log'}], '...er_attributes': {'serverHost': 'test_basic-3.10-macos-11-9180500811-1-1716320620'}, 'verify_server_certificate': False}

    def test_basic(
        scalyr_api_key,
        scalyr_server,
        scalyr_api_read_key,
        server_host,
        agent_paths,
        agent_commander,
        default_config,
    ):
        """
        Perform some basic checks to running agent.
        Verify agent log for errors, write test messages to a test log file and check if
        those messages are reached the Scalyr server.
        """
        upload_test_log_path = agent_paths.logs_dir / "test.log"
    
        timeout_tracker = TimeoutTracker(200)
    
        default_config["logs"] = [
            {"path": str(upload_test_log_path), "attributes": {"parser": "json"}}
        ]
    
        agent_paths.agent_config_path.write_text(json.dumps(default_config))
    
>       agent_commander.start_and_wait()

#x1B[1m#x1B[31mtests/end_to_end_tests/test_from_source/test_from_source.py#x1B[0m:73: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
#x1B[1m#x1B[31mtests/end_to_end_tests/tools.py#x1B[0m:98: in start_and_wait
    self.start(env=env)
#x1B[1m#x1B[31mtests/end_to_end_tests/tools.py#x1B[0m:90: in start
    self._check_call_command(cmd, env=env)
#x1B[1m#x1B[31mtests/end_to_end_tests/tools.py#x1B[0m:66: in _check_call_command
    subprocess.check_call([*self.executable_args, *command_args], **kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

popenargs = (['/Users/runner/work/scalyr-agent-2/scalyr-agent-2/.tox/py310-end2end/bin/python', '/Users/runner/work/scalyr-agent-2/scalyr-agent-2/scalyr_agent/agent_main.py', 'start'],)
kwargs = {'env': None}, retcode = 1
cmd = ['/Users/runner/work/scalyr-agent-2/scalyr-agent-2/.tox/py310-end2end/bin/python', '/Users/runner/work/scalyr-agent-2/scalyr-agent-2/scalyr_agent/agent_main.py', 'start']

    def check_call(*popenargs, **kwargs):
        """Run command with arguments.  Wait for command to complete.  If
        the exit code was zero then return, otherwise raise
        CalledProcessError.  The CalledProcessError object will have the
        return code in the returncode attribute.
    
        The arguments are the same as for the call function.  Example:
    
        check_call(["ls", "-l"])
        """
        retcode = call(*popenargs, **kwargs)
        if retcode:
            cmd = kwargs.get("args")
            if cmd is None:
                cmd = popenargs[0]
>           raise CalledProcessError(retcode, cmd)
#x1B[1m#x1B[31mE           subprocess.CalledProcessError: Command '['/Users/runner/work/scalyr-agent-2/scalyr-agent-2/.tox/py310-end2end/bin/python', '/Users/runner/work/scalyr-agent-2/scalyr-agent-2/scalyr_agent/agent_main.py', 'start']' returned non-zero exit status 1.#x1B[0m

#x1B[1m#x1B[31m../../../hostedtoolcache/Python/3.10.13/x64/lib/python3.10/subprocess.py#x1B[0m:369: CalledProcessError

subprocess.CalledProcessError: Command '['/Users/runner/work/scalyr-agent-2/scalyr-agent-2/.tox/py310-end2end/bin/python', '/Users/runner/work/scalyr-agent-2/scalyr-agent-2/scalyr_agent/agent_main.py', 'start']' returned non-zero exit status 1.
scalyr_api_key = '', scalyr_server = 'agent.scalyr.com'
scalyr_api_read_key = ''
server_host = 'test_rate_limited-3.10-macos-11-9180500811-1-1716320620'
agent_paths = AgentPaths(configs_dir=PosixPath('/Users/runner/scalyr-agent-dev/config'), logs_dir=PosixPath('/Users/runner/scalyr-ag...nner/scalyr-agent-dev/log/agent.log'), agent_config_path=PosixPath('/Users/runner/scalyr-agent-dev/config/agent.json'))
agent_commander = <tests.end_to_end_tests.tools.AgentCommander object at 0x10cf52350>
default_config = {'api_key': '', 'logs': [{'attributes': {'parser': 'json'}, 'path': '/Users/runner/scalyr-agent-dev/log/rate_limited_d...ibutes': {'serverHost': 'test_rate_limited-3.10-macos-11-9180500811-1-1716320620'}, 'verify_server_certificate': False}

    def test_rate_limited(
        scalyr_api_key,
        scalyr_server,
        scalyr_api_read_key,
        server_host,
        agent_paths,
        agent_commander,
        default_config,
    ):
        """
        Writes 5000 large lines to data.log, then waits until it detects at least one such message in
        Scalyr, then waits a set time before checking how many lines have been uploaded. The intent of this is to test
        rate limiting, and as such the agent must be configured with a rate limiting options.
        """
        upload_test_log_path = agent_paths.logs_dir / "rate_limited_data.log"
    
        timeout_tracker = TimeoutTracker(200)
    
        default_config["logs"] = [
            {"path": str(upload_test_log_path), "attributes": {"parser": "json"}}
        ]
    
        agent_paths.agent_config_path.write_text(json.dumps(default_config))
    
        # This value should match the rate limit configured for the agent when running this test
        rate_limit_bytes_per_second = 500000
    
        upload_test_log_path.touch()
    
>       agent_commander.start_and_wait(
            logger=logger,
            env={
                "SCALYR_MAX_SEND_RATE_ENFORCEMENT": "500KB/s",
                "SCALYR_DISABLE_MAX_SEND_RATE_ENFORCEMENT_OVERRIDES": "true",
                "SCALYR_MIN_ALLOWED_REQUEST_SIZE": "100",
                "SCALYR_MAX_ALLOWED_REQUEST_SIZE": "500000",
                "SCALYR_MIN_REQUEST_SPACING_INTERVAL": "0.0",
                "SCALYR_MAX_REQUEST_SPACING_INTERVAL": "0.5",
            },
        )

#x1B[1m#x1B[31mtests/end_to_end_tests/test_from_source/test_from_source.py#x1B[0m:132: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
#x1B[1m#x1B[31mtests/end_to_end_tests/tools.py#x1B[0m:98: in start_and_wait
    self.start(env=env)
#x1B[1m#x1B[31mtests/end_to_end_tests/tools.py#x1B[0m:90: in start
    self._check_call_command(cmd, env=env)
#x1B[1m#x1B[31mtests/end_to_end_tests/tools.py#x1B[0m:66: in _check_call_command
    subprocess.check_call([*self.executable_args, *command_args], **kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

popenargs = (['/Users/runner/work/scalyr-agent-2/scalyr-agent-2/.tox/py310-end2end/bin/python', '/Users/runner/work/scalyr-agent-2/scalyr-agent-2/scalyr_agent/agent_main.py', 'start'],)
kwargs = {'env': {'SCALYR_DISABLE_MAX_SEND_RATE_ENFORCEMENT_OVERRIDES': 'true', 'SCALYR_MAX_ALLOWED_REQUEST_SIZE': '500000', 'SCALYR_MAX_REQUEST_SPACING_INTERVAL': '0.5', 'SCALYR_MAX_SEND_RATE_ENFORCEMENT': '500KB/s', ...}}
retcode = 1
cmd = ['/Users/runner/work/scalyr-agent-2/scalyr-agent-2/.tox/py310-end2end/bin/python', '/Users/runner/work/scalyr-agent-2/scalyr-agent-2/scalyr_agent/agent_main.py', 'start']

    def check_call(*popenargs, **kwargs):
        """Run command with arguments.  Wait for command to complete.  If
        the exit code was zero then return, otherwise raise
        CalledProcessError.  The CalledProcessError object will have the
        return code in the returncode attribute.
    
        The arguments are the same as for the call function.  Example:
    
        check_call(["ls", "-l"])
        """
        retcode = call(*popenargs, **kwargs)
        if retcode:
            cmd = kwargs.get("args")
            if cmd is None:
                cmd = popenargs[0]
>           raise CalledProcessError(retcode, cmd)
#x1B[1m#x1B[31mE           subprocess.CalledProcessError: Command '['/Users/runner/work/scalyr-agent-2/scalyr-agent-2/.tox/py310-end2end/bin/python', '/Users/runner/work/scalyr-agent-2/scalyr-agent-2/scalyr_agent/agent_main.py', 'start']' returned non-zero exit status 1.#x1B[0m

#x1B[1m#x1B[31m../../../hostedtoolcache/Python/3.10.13/x64/lib/python3.10/subprocess.py#x1B[0m:369: CalledProcessError

subprocess.CalledProcessError: Command '['/Users/runner/work/scalyr-agent-2/scalyr-agent-2/.tox/py310-end2end/bin/python', '/Users/runner/work/scalyr-agent-2/scalyr-agent-2/scalyr_agent/agent_main.py', 'start']' returned non-zero exit status 1.
scalyr_api_key = '', scalyr_server = 'agent.scalyr.com'
scalyr_api_read_key = ''
server_host = 'test_with_failing_essential_monitor-3.10-macos-11-9180500811-1-1716320620'
agent_paths = AgentPaths(configs_dir=PosixPath('/Users/runner/scalyr-agent-dev/config'), logs_dir=PosixPath('/Users/runner/scalyr-ag...nner/scalyr-agent-dev/log/agent.log'), agent_config_path=PosixPath('/Users/runner/scalyr-agent-dev/config/agent.json'))
agent_commander = <tests.end_to_end_tests.tools.AgentCommander object at 0x10cf52350>
default_config = {'additional_monitor_module_paths': '/Users/runner/work/scalyr-agent-2/scalyr-agent-2/tests/end_to_end_tests/test_from... 'server_attributes': {'serverHost': 'test_with_failing_essential_monitor-3.10-macos-11-9180500811-1-1716320620'}, ...}

    @pytest.mark.timeout(40)
    def test_with_failing_essential_monitor(
        scalyr_api_key,
        scalyr_server,
        scalyr_api_read_key,
        server_host,
        agent_paths,
        agent_commander,
        default_config,
    ):
        """
        Check that special 'stop_agent_on_failure' directive in a monitor's config
        make the whole agent fail if the monitor fails too.
        """
        test_monitors_path = pl.Path(__file__).parent / "fixtures"
    
        default_config.update(
            {
                "additional_monitor_module_paths": str(test_monitors_path),
                "monitors": [
                    {
                        "module": "monitors.essential_failing_monitor",
                        "id": "essential",
                        "stop_agent_on_failure": True,
                    }
                ],
            }
        )
    
        agent_paths.agent_config_path.write_text(json.dumps(default_config))
    
        logger.info("Starting agent.")
>       agent_commander.start()

#x1B[1m#x1B[31mtests/end_to_end_tests/test_from_source/test_from_source.py#x1B[0m:276: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
#x1B[1m#x1B[31mtests/end_to_end_tests/tools.py#x1B[0m:90: in start
    self._check_call_command(cmd, env=env)
#x1B[1m#x1B[31mtests/end_to_end_tests/tools.py#x1B[0m:66: in _check_call_command
    subprocess.check_call([*self.executable_args, *command_args], **kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

popenargs = (['/Users/runner/work/scalyr-agent-2/scalyr-agent-2/.tox/py310-end2end/bin/python', '/Users/runner/work/scalyr-agent-2/scalyr-agent-2/scalyr_agent/agent_main.py', 'start'],)
kwargs = {'env': None}, retcode = 1
cmd = ['/Users/runner/work/scalyr-agent-2/scalyr-agent-2/.tox/py310-end2end/bin/python', '/Users/runner/work/scalyr-agent-2/scalyr-agent-2/scalyr_agent/agent_main.py', 'start']

    def check_call(*popenargs, **kwargs):
        """Run command with arguments.  Wait for command to complete.  If
        the exit code was zero then return, otherwise raise
        CalledProcessError.  The CalledProcessError object will have the
        return code in the returncode attribute.
    
        The arguments are the same as for the call function.  Example:
    
        check_call(["ls", "-l"])
        """
        retcode = call(*popenargs, **kwargs)
        if retcode:
            cmd = kwargs.get("args")
            if cmd is None:
                cmd = popenargs[0]
>           raise CalledProcessError(retcode, cmd)
#x1B[1m#x1B[31mE           subprocess.CalledProcessError: Command '['/Users/runner/work/scalyr-agent-2/scalyr-agent-2/.tox/py310-end2end/bin/python', '/Users/runner/work/scalyr-agent-2/scalyr-agent-2/scalyr_agent/agent_main.py', 'start']' returned non-zero exit status 1.#x1B[0m

#x1B[1m#x1B[31m../../../hostedtoolcache/Python/3.10.13/x64/lib/python3.10/subprocess.py#x1B[0m:369: CalledProcessError

subprocess.CalledProcessError: Command '['/Users/runner/work/scalyr-agent-2/scalyr-agent-2/.tox/py310-end2end/bin/python', '/Users/runner/work/scalyr-agent-2/scalyr-agent-2/scalyr_agent/agent_main.py', 'start']' returned non-zero exit status 1.
scalyr_api_key = '', scalyr_server = 'agent.scalyr.com'
scalyr_api_read_key = ''
server_host = 'test_with_failing_non_essential_monitors-3.10-macos-11-9180500811-1-1716320620'
agent_paths = AgentPaths(configs_dir=PosixPath('/Users/runner/scalyr-agent-dev/config'), logs_dir=PosixPath('/Users/runner/scalyr-ag...nner/scalyr-agent-dev/log/agent.log'), agent_config_path=PosixPath('/Users/runner/scalyr-agent-dev/config/agent.json'))
agent_commander = <tests.end_to_end_tests.tools.AgentCommander object at 0x10cf52350>
default_config = {'additional_monitor_module_paths': '/Users/runner/work/scalyr-agent-2/scalyr-agent-2/tests/end_to_end_tests/test_from...ver_attributes': {'serverHost': 'test_with_failing_non_essential_monitors-3.10-macos-11-9180500811-1-1716320620'}, ...}

    @pytest.mark.timeout(40)
    def test_with_failing_non_essential_monitors(
        scalyr_api_key,
        scalyr_server,
        scalyr_api_read_key,
        server_host,
        agent_paths,
        agent_commander,
        default_config,
    ):
        """
        Test the 'stop_agent_on_failure' option is set to False or default.
        Agent, for the backward compatibility reasons, must not fail with those "non-essential" monitors.
        """
        test_monitors_path = pl.Path(__file__).parent / "fixtures"
    
        default_config.update(
            {
                "additional_monitor_module_paths": str(test_monitors_path),
                "monitors": [
                    {
                        "module": "monitors.essential_failing_monitor",
                        "id": "not_essential",
                        "stop_agent_on_failure": False,
                    },
                    {
                        "module": "monitors.essential_failing_monitor",
                        "id": "not_essential_default",
                    },
                ],
            }
        )
    
        agent_paths.agent_config_path.write_text(json.dumps(default_config))
    
        logger.info("Starting agent.")
>       agent_commander.start()

#x1B[1m#x1B[31mtests/end_to_end_tests/test_from_source/test_from_source.py#x1B[0m:349: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
#x1B[1m#x1B[31mtests/end_to_end_tests/tools.py#x1B[0m:90: in start
    self._check_call_command(cmd, env=env)
#x1B[1m#x1B[31mtests/end_to_end_tests/tools.py#x1B[0m:66: in _check_call_command
    subprocess.check_call([*self.executable_args, *command_args], **kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

popenargs = (['/Users/runner/work/scalyr-agent-2/scalyr-agent-2/.tox/py310-end2end/bin/python', '/Users/runner/work/scalyr-agent-2/scalyr-agent-2/scalyr_agent/agent_main.py', 'start'],)
kwargs = {'env': None}, retcode = 1
cmd = ['/Users/runner/work/scalyr-agent-2/scalyr-agent-2/.tox/py310-end2end/bin/python', '/Users/runner/work/scalyr-agent-2/scalyr-agent-2/scalyr_agent/agent_main.py', 'start']

    def check_call(*popenargs, **kwargs):
        """Run command with arguments.  Wait for command to complete.  If
        the exit code was zero then return, otherwise raise
        CalledProcessError.  The CalledProcessError object will have the
        return code in the returncode attribute.
    
        The arguments are the same as for the call function.  Example:
    
        check_call(["ls", "-l"])
        """
        retcode = call(*popenargs, **kwargs)
        if retcode:
            cmd = kwargs.get("args")
            if cmd is None:
                cmd = popenargs[0]
>           raise CalledProcessError(retcode, cmd)
#x1B[1m#x1B[31mE           subprocess.CalledProcessError: Command '['/Users/runner/work/scalyr-agent-2/scalyr-agent-2/.tox/py310-end2end/bin/python', '/Users/runner/work/scalyr-agent-2/scalyr-agent-2/scalyr_agent/agent_main.py', 'start']' returned non-zero exit status 1.#x1B[0m

#x1B[1m#x1B[31m../../../hostedtoolcache/Python/3.10.13/x64/lib/python3.10/subprocess.py#x1B[0m:369: CalledProcessError