Improves memory limit tests (#1057)

integration/README.md

@@ -4,18 +4,14 @@ Our integration tests currently require Python 3.6.
 All dependencies are specified in [`requirements.txt`](./requirements.txt),
 and can be installed automatically via `pip`:
 
-Before running these tests, please make sure you've followed the instructions
-in the [Cook Readme](../README.md) Quickstart section. In particular, to run
-the integration tests in docker, you need to have minimesos running, and the
-cook scheduler running within docker. You also need to have 'jq' installed
-for json queries.
-
-
 ```bash
 $ pip install -r requirements.txt
 ```
 
-To run the tests on a local cook install, run
+Before running these tests, please make sure you've followed the instructions in the [Cook Readme](../README.md) Quickstart section.
+In particular, to run the integration tests in docker, you need to have minimesos running, and the Cook Scheduler running within docker. You also need to have `jq` installed.
+
+To run the tests on a local Cook install, run:
 
 ```bash
 $ pytest
@@ -29,13 +25,23 @@ If you want to run a single test and see the log messages as they occur, you can
 $ ./bin/only-run test_basic_submit
 ```
 
-The `./bin/only-run` helper script invokes pytest internally.
+The `./bin/only-run` helper script invokes `pytest` internally.
 The above command is roughly equivalent to the following pytest invocation:
 
 ```bash
 $ pytest -v -n0 --capture=no tests/cook/test_basic.py::CookTest::test_basic_submit
 ```
 
+## Memory limit tests
+
+There are a few tests marked as `memlimit` which test how Cook Scheduler behaves when a task exceeds its memory limit.
+Unfortunately, if you're running on macos and you're not using minimesos, these tests will fail, because the Mesos agent flags that dictate this behavior are not available (specifically, `isolation` cannot be set to `cgroups/mem`).
+To skip these tests, simply run:
+
+```bash
+$ pytest -m 'not memlimit'
+```
+
 ## Multi-scheduler tests
 
 The [test_multi_cluster.py](tests/cook/test_multi_cluster.py) file contains integration tests that rely on two separate running schedulers.

integration/tests/cook/test_basic.py

@@ -483,9 +483,10 @@ def memory_limit_exceeded_helper(self, command, executor_type):
             job = util.wait_for_job(self.cook_url, job_uuid, 'completed')
             job_details = f"Job details: {json.dumps(job, sort_keys=True)}"
             self.assertEqual('failed', job['state'], job_details)
-            self.assertEqual(1, len(job['instances']), job_details)
-            instance = job['instances'][0]
+            self.assertLessEqual(1, len(job['instances']), job_details)
+            instance = job['instances'][-1]
             instance_details = json.dumps(instance, sort_keys=True)
+            self.logger.debug('instance: %s' % instance)
             # did the job fail as expected?
             self.assertEqual(executor_type, instance['executor'], instance_details)
             self.assertEqual('failed', instance['status'], instance_details)
@@ -508,14 +509,12 @@ def memory_limit_exceeded_helper(self, command, executor_type):
             util.kill_jobs(self.cook_url, [job_uuid])
 
     @pytest.mark.memlimit
-    @unittest.skipUnless(util.continuous_integration(), "Doesn't work in our local test environments")
     @unittest.skipUnless(util.is_cook_executor_in_use(), 'Test assumes the Cook Executor is in use')
     def test_memory_limit_exceeded_cook_python(self):
         command = self.memory_limit_python_command()
         self.memory_limit_exceeded_helper(command, 'cook')
 
     @pytest.mark.memlimit
-    @unittest.skipUnless(util.continuous_integration(), "Doesn't work in our local test environments")
     def test_memory_limit_exceeded_mesos_python(self):
         command = self.memory_limit_python_command()
         self.memory_limit_exceeded_helper(command, 'mesos')
@@ -527,7 +526,6 @@ def test_memory_limit_exceeded_cook_script(self):
         self.memory_limit_exceeded_helper(command, 'cook')
 
     @pytest.mark.memlimit
-    @unittest.skipUnless(util.continuous_integration(), "Doesn't work in our local test environments")
     def test_memory_limit_exceeded_mesos_script(self):
         command = self.memory_limit_script_command()
         self.memory_limit_exceeded_helper(command, 'mesos')