Bug 1169320 - Tests against the JobLoader

tests/etl/test_job_loader.py

@@ -0,0 +1,103 @@
+import copy
+import pytest
+
+from treeherder.etl.job_loader import JobLoader
+from treeherder.model.derived.artifacts import ArtifactsModel
+
+
+@pytest.fixture
+def first_job(sample_data, test_project, result_set_stored):
+    revision = result_set_stored[0]["revisions"][0]["revision"]
+    job = copy.deepcopy(sample_data.pulse_jobs[0])
+    job["origin"]["project"] = test_project
+    job["origin"]["revision"] = revision
+    return job
+
+
+def test_ingest_pulse_job(sample_data, test_project, jm, result_set_stored):
+    """
+    Ingest a job through the JSON Schema validated JobLoader used by Pulse
+    """
+    revision = result_set_stored[0]["revisions"][0]["revision"]
+    sample_jobs = sample_data.pulse_jobs
+    for job in sample_jobs:
+        job["origin"]["project"] = test_project
+        job["origin"]["revision"] = revision
+
+    jl = JobLoader()
+    jl.process_job_list(sample_jobs, raise_errors=True)
+
+    jobs = jm.get_job_list(0, 10)
+    assert len(jobs) == 3
+
+    logs = jm.get_job_log_url_list([jobs[0]["id"]])
+    assert len(logs) == 1
+    with ArtifactsModel(test_project) as am:
+        artifacts = am.get_job_artifact_list(0, 10)
+        assert len(artifacts) == 2
+
+
+def test_transition_pending_running_complete(first_job, jm):
+    jl = JobLoader()
+
+    change_state_result(first_job, jl, jm, "pending", "unknown", "pending", "unknown")
+    change_state_result(first_job, jl, jm, "running", "unknown", "running", "unknown")
+    change_state_result(first_job, jl, jm, "completed", "fail", "completed", "testfailed")
+
+
+def test_transition_complete_pending_stays_complete(first_job, jm):
+    jl = JobLoader()
+
+    change_state_result(first_job, jl, jm, "completed", "fail", "completed", "testfailed")
+    change_state_result(first_job, jl, jm, "pending", "unknown", "completed", "testfailed")
+
+
+def test_transition_complete_running_stays_complete(first_job, jm):
+    jl = JobLoader()
+
+    change_state_result(first_job, jl, jm, "completed", "fail", "completed", "testfailed")
+    change_state_result(first_job, jl, jm, "running", "unknown", "completed", "testfailed")
+
+
+def test_transition_running_pending_stays_running(first_job, jm):
+    jl = JobLoader()
+
+    change_state_result(first_job, jl, jm, "running", "unknown", "running", "unknown")
+    change_state_result(first_job, jl, jm, "pending", "unknown", "running", "unknown")
+
+
+def test_transition_pending_retry_fail_stays_retry(first_job, jm):
+    jl = JobLoader()
+
+    change_state_result(first_job, jl, jm, "pending", "unknown", "pending", "unknown")
+    first_job["isRetried"] = True
+    change_state_result(first_job, jl, jm, "completed", "fail", "completed", "retry")
+    first_job["isRetried"] = False
+    change_state_result(first_job, jl, jm, "completed", "fail", "completed", "retry")
+
+
+def test_skip_unscheduled(first_job, jm):
+    jl = JobLoader()
+    first_job["state"] = "unscheduled"
+    jl.process_job_list([first_job], raise_errors=True)
+
+    jobs = jm.get_job_list(0, 10)
+    assert len(jobs) == 0
+
+
+def change_state_result(job, job_loader, jm, new_state, new_result, exp_state, exp_result):
+    # make a copy so we can modify it and not affect other tests
+    job = copy.deepcopy(job)
+    job["state"] = new_state
+    job["result"] = new_result
+    if new_state == 'pending':
+        # pending jobs wouldn't have logs and our store_job_data doesn't
+        # support it.
+        del job['logs']
+
+    job_loader.process_job_list([job], raise_errors=True)
+
+    jobs = jm.get_job_list(0, 10)
+    assert len(jobs) == 1
+    assert jobs[0]['state'] == exp_state
+    assert jobs[0]['result'] == exp_result

tests/etl/test_job_schema.py

@@ -0,0 +1,29 @@
+import pytest
+import jsonschema
+
+from treeherder.etl.schema import job_json_schema
+
+# The test data in this file are a representative sample-set from
+# production Treeherder
+
+
+@pytest.mark.parametrize("group_symbol", ['?', 'A', 'Aries', 'Buri/Hamac', 'L10n', 'M-e10s'])
+def test_group_symbols(sample_data, group_symbol):
+    """
+    Validate jobs against the schema with different group_symbol values
+    """
+    job = sample_data.pulse_jobs[0]
+    job["origin"]["project"] = "proj"
+    job["display"]["groupSymbol"] = group_symbol
+    jsonschema.validate(job, job_json_schema)
+
+
+@pytest.mark.parametrize("job_symbol", ['1.1g', '1g', '20', 'A', 'GBI10', 'en-US-1'])
+def test_job_symbols(sample_data, job_symbol):
+    """
+        Validate jobs against the schema with different job_symbol values
+    """
+    job = sample_data.pulse_jobs[0]
+    job["origin"]["project"] = "proj"
+    job["display"]["jobSymbol"] = job_symbol
+    jsonschema.validate(job, job_json_schema)

tests/sample_data/pulse_consumer/job_data.json

@@ -0,0 +1,150 @@
+[
+  {
+    "jobGuid": "d22c74d4aa6d2a1dcba96d95dccbd5fdca70cf33",
+    "origin": {
+      "kind": "hg.mozilla.org",
+      "project": "set by test",
+      "revision": "set by test"
+    },
+
+    "display": {
+      "jobSymbol": "P1",
+      "jobName": "Pulse Ingestion Test",
+      "groupSymbol": "PI",
+      "groupName": "PulseIngestion"
+    },
+
+    "state": "completed",
+    "result": "success",
+    "jobKind": "test",
+
+    "runMachine": {
+      "name": "bld-linux64-ec2-104",
+      "platform": "linux64",
+      "os": "linux",
+      "architecture": "x86_64"
+    },
+    "buildMachine": {
+      "name": "bld-linux64-ec2-104",
+      "platform": "linux64",
+      "os": "linux",
+      "architecture": "x86_64"
+    },
+
+    "who": "thedude@lebowski.com",
+    "reason": "scheduler",
+    "productName": "Oscillation Overthruster",
+
+    "timeScheduled": "2014-12-19T16:39:57-08:00",
+    "timeStarted": "2014-12-19T17:39:57-08:00",
+    "timeCompleted": "2014-12-19T18:39:57-08:00",
+
+    "logs": [
+      {
+        "url": "http://ftp.mozilla.org/pub/mozilla.org/spidermonkey/tinderbox-builds/mozilla-inbound-linux64/mozilla-inbound_linux64_spidermonkey-warnaserr-bm57-build1-build352.txt.gz",
+        "parseStatus": "parsed",
+        "name": "builds-4h"
+      }
+    ],
+    "artifacts": [
+      {
+        "type": "json",
+        "name": "Artie Fact",
+        "blob": "{\"some\": \"value\"}"
+      }
+    ]
+  },
+  {
+    "jobGuid": "d22c74d4aa6d2a1dcba96d95dccbd5fdca70cf34",
+    "origin": {
+      "kind": "hg.mozilla.org",
+      "project": "set by test",
+      "revision": "set by test"
+    },
+
+    "display": {
+      "jobSymbol": "P2",
+      "jobName": "Pulse Ingestion Test",
+      "groupSymbol": "?",
+      "groupName": "unknown"
+    },
+
+    "state": "running",
+    "jobKind": "test",
+
+    "runMachine": {
+      "name": "bld-linux64-ec2-104",
+      "platform": "linux64",
+      "os": "linux",
+      "architecture": "x86_64"
+    },
+    "buildMachine": {
+      "name": "bld-linux64-ec2-104",
+      "platform": "linux64",
+      "os": "linux",
+      "architecture": "x86_64"
+    },
+
+    "who": "thedude@lebowski.com",
+    "reason": "scheduler",
+    "productName": "Oscillation Overthruster",
+
+    "timeScheduled": "2014-12-19T16:39:57-08:00",
+    "timeStarted": "2014-12-19T17:39:57-08:00",
+    "timeCompleted": "2014-12-19T18:39:57-08:00"
+  },
+  {
+    "jobGuid": "d22c74d4aa6d2a1dcba96d95dccbd5fdca70cf35",
+    "origin": {
+      "kind": "hg.mozilla.org",
+      "project": "set by test",
+      "revision": "set by test"
+    },
+
+    "display": {
+      "jobSymbol": "P3",
+      "jobName": "Pulse Ingestion Test",
+      "groupSymbol": "PI",
+      "groupName": "PulseIngestion"
+    },
+
+    "state": "completed",
+    "result": "fail",
+    "jobKind": "test",
+
+    "runMachine": {
+      "name": "bld-linux64-ec2-104",
+      "platform": "linux64",
+      "os": "linux",
+      "architecture": "x86_64"
+    },
+    "buildMachine": {
+      "name": "bld-linux64-ec2-104",
+      "platform": "linux64",
+      "os": "linux",
+      "architecture": "x86_64"
+    },
+
+    "who": "thedude@lebowski.com",
+    "reason": "scheduler",
+
+    "timeScheduled": "2014-12-19T16:39:57-08:00",
+    "timeStarted": "2014-12-19T17:39:57-08:00",
+    "timeCompleted": "2014-12-19T18:39:57-08:00",
+
+    "logs": [
+      {
+        "url": "http://ftp.mozilla.org/pub/mozilla.org/spidermonkey/tinderbox-builds/mozilla-inbound-linux64/mozilla-inbound_linux64_spidermonkey-warnaserr-bm57-build1-build352.txt.gz",
+        "parseStatus": "parsed",
+        "name": "builds-4h"
+      }
+    ],
+    "artifacts": [
+      {
+        "type": "json",
+        "name": "Artie Fact",
+        "blob": "{\"some\": \"value\"}"
+      }
+    ]
+  }
+]

tests/sampledata.py

@@ -48,6 +48,7 @@ def __init__(self):
         self.logs_dir = "{0}/sample_data/logs".format(
             os.path.dirname(__file__)
         )
+
         self.performance_logs_dir = "{0}/sample_data/artifacts/performance/perf_logs".format(
             os.path.dirname(__file__)
         )
@@ -64,6 +65,10 @@ def __init__(self):
                   os.path.dirname(__file__))) as f:
             self.text_log_summary = json.load(f)
 
+        with open("{0}/sample_data/pulse_consumer/job_data.json".format(
+                  os.path.dirname(__file__))) as f:
+            self.pulse_jobs = json.load(f)
+
         self.job_data = []
         self.resultset_data = []