Hefty tests

Hefty tests are a method of defining a test that should not be run on every
patch of a series.  Testing every patch in a series can be a large resource
commitment, and in a lot of cases should just be a smoke test for
bisectability.

If "hefty = true" is set in a job, then it will only be run on "complete"
series - a single patch, or the last patch in a series and its dependencies.

Signed-off-by: Russell Currey <ruscur@russell.cc>
[ajd: rebase on serde and patchwork API changes]
Signed-off-by: Andrew Donnellan <andrew.donnellan@au1.ibm.com>

examples/openpower.toml

@@ -67,11 +67,13 @@ token = "33333333333333333333333333333333"
         remote = "GIT_REPO"
         branch = "GIT_REF"
         artifact = "snowpatch.txt"
+        hefty = true
         DEFCONFIG_TO_USE = "pseries_le_defconfig"
 
         [[projects.linuxppc-dev.jobs]]
         job = "linux-build-manual"
         remote = "GIT_REPO"
         branch = "GIT_REF"
         artifact = "snowpatch.txt"
+        hefty = false
         DEFCONFIG_TO_USE = "ppc64le_defconfig"

src/main.rs

@@ -103,7 +103,7 @@ struct Args {
 }
 
 fn run_tests(settings: &Config, client: Arc<Client>, project: &Project, tag: &str,
-             branch_name: &str) -> Vec<TestResult> {
+             branch_name: &str, hefty_tests: bool) -> Vec<TestResult> {
     let mut results: Vec<TestResult> = Vec::new();
     let jenkins = JenkinsBackend {
         base_url: settings.jenkins.url.clone(),
@@ -113,6 +113,10 @@ fn run_tests(settings: &Config, client: Arc<Client>, project: &Project, tag: &st
     };
     let project = project.clone();
     for job in &project.jobs {
+        if !hefty_tests && job.hefty {
+            debug!("Skipping hefty test {}", job.title);
+            continue;
+        }
         let mut jenkins_params = Vec::<(&str, &str)>::new();
         for (param_name, param_value) in job.parameters.iter() {
             // TODO(ajd): do this more neatly
@@ -149,7 +153,8 @@ fn run_tests(settings: &Config, client: Arc<Client>, project: &Project, tag: &st
     results
 }
 
-fn test_patch(settings: &Config, client: &Arc<Client>, project: &Project, path: &Path) -> Vec<TestResult> {
+fn test_patch(settings: &Config, client: &Arc<Client>, project: &Project,
+              path: &Path, hefty_tests: bool) -> Vec<TestResult> {
     let repo = project.get_repo().unwrap();
     let mut results: Vec<TestResult> = Vec::new();
     if !path.is_file() {
@@ -235,7 +240,8 @@ fn test_patch(settings: &Config, client: &Arc<Client>, project: &Project, path:
 
         // We've set up a remote branch, time to kick off tests
         let test = thread::Builder::new().name(tag.to_string()).spawn(move || {
-            run_tests(&settings_clone, client, &project, &tag, &branch_name)
+            run_tests(&settings_clone, client, &project, &tag, &branch_name,
+            hefty_tests)
         }).unwrap();
         results.append(&mut test.join().unwrap());
 
@@ -317,7 +323,7 @@ fn main() {
         match settings.projects.get(&args.flag_project) {
             None => panic!("Couldn't find project {}", args.flag_project),
             Some(project) => {
-                test_patch(&settings, &client, project, patch);
+                test_patch(&settings, &client, project, patch, true);
             }
         }
 
@@ -336,7 +342,7 @@ fn main() {
                 } else {
                     patchwork.get_patch_mbox(&patch)
                 };
-                test_patch(&settings, &client, project, &mbox);
+                test_patch(&settings, &client, project, &mbox, true);
             }
         }
         return;
@@ -354,7 +360,7 @@ fn main() {
             Some(project) => {
                 let dependencies = patchwork.get_patch_dependencies(&patch);
                 let mbox = patchwork.get_patches_mbox(dependencies);
-                test_patch(&settings, &client, project, &mbox);
+                test_patch(&settings, &client, project, &mbox, true);
             }
         }
         return;
@@ -401,6 +407,7 @@ fn main() {
                 },
                 Some(project) => {
                     // TODO(ajd): Refactor this.
+                    let hefty_tests;
                     let mbox = if patch.has_series() {
                         debug!("Patch {} has a series at {}!", &patch.name, &patch.series[0].url);
                         let series = patchwork.get_series_by_url(&patch.series[0].url);
@@ -411,19 +418,22 @@ fn main() {
                                     continue;
                                 }
                                 let dependencies = patchwork.get_patch_dependencies(&patch);
+                                hefty_tests = dependencies.len() == series.patches.len();
                                 patchwork.get_patches_mbox(dependencies)
 
                             },
                             Err(e) => {
                                 debug!("Series is not OK: {}", e);
+                                hefty_tests = true;
                                 patchwork.get_patch_mbox(&patch)
                             }
                         }
                     } else {
+                        hefty_tests = true;
                         patchwork.get_patch_mbox(&patch)
                     };
 
-                    let results = test_patch(&settings, &client, project, &mbox);
+                    let results = test_patch(&settings, &client, project, &mbox, hefty_tests);
 
                     // Delete the temporary directory with the patch in it
                     fs::remove_dir_all(mbox.parent().unwrap()).unwrap_or_else(

src/settings.rs

@@ -80,6 +80,7 @@ pub struct Job {
     pub title: String,
     pub remote: String,
     pub branch: String,
+    pub hefty: bool,
     pub parameters: BTreeMap<String, String>,
 }
 
@@ -103,6 +104,7 @@ impl<'de> Deserialize<'de> for Job {
                 let mut title = None;
                 let mut remote = None;
                 let mut branch = None;
+                let mut hefty = None;
                 let mut parameters = BTreeMap::new();
                 while let Some(key) = map.next_key::<String>()? {
                     match key.as_str() {
@@ -130,6 +132,12 @@ impl<'de> Deserialize<'de> for Job {
                             }
                             branch = Some(map.next_value()?);
                         }
+                        "hefty" => {
+                            if hefty.is_some() {
+                                return Err(de::Error::duplicate_field("hefty"));
+                            }
+                            hefty = Some(map.next_value()?);
+                        }
                         _ => {
                             parameters.insert(key, map.next_value()?);
                         }
@@ -140,12 +148,14 @@ impl<'de> Deserialize<'de> for Job {
                 let remote = remote.ok_or_else(|| de::Error::missing_field("remote"))?;
                 let branch = branch.ok_or_else(|| de::Error::missing_field("branch"))?;
                 let title = title.unwrap_or(job.clone());
+                let hefty = hefty.unwrap_or(false);
 
                 Ok(Job {
                     job: job,
                     title: title,
                     remote: remote,
                     branch: branch,
+                    hefty: hefty,
                     parameters: parameters,
                 })
             }