New flakiness dashboard should generate JSON in the background process

https://bugs.webkit.org/show_bug.cgi?id=123797 Reviewed by Alexey Proskuryakov. * config.json: Add defaultBuildWaitInterval used by process-builds.php. * init-database.sql: Add is_process column to builds table so that we can track of "unprocessed" builds. * public/admin/process-builds.php: Added. (process_latest_five_builds): Update flakiness states and generate JSONs for the latest five builds. We go backwards in the time so that we don't end up infinite looping over a single build that fails. (main): Call processed_builds with an exponential back off. * public/api/report.php: (store_results): No need to return build or builder ids. (main): Don't update flakiness states or generate JSONs. * public/include/test-results.php: (add_build): Take the slave name; it doesn't make any sense to create a build without it and later updating it in store_test_results. (store_test_results): Once new test results are added, explicitly mark this build as unprocessed. Canonical link: https://commits.webkit.org/142130@main git-svn-id: https://svn.webkit.org/repository/webkit/trunk@158814 268f45cc-cd09-0410-ab3c-d52691b4dbfc
WebKit · Nov 7, 2013 · 26045b3 · 26045b3
1 parent 05b5216
commit 26045b3
Show file tree

Hide file tree

Showing 6 changed files with 99 additions and 29 deletions.
diff --git a/Websites/test-results/ChangeLog b/Websites/test-results/ChangeLog
@@ -1,3 +1,27 @@
+2013-11-06  Ryosuke Niwa  <rniwa@webkit.org>
+
+        New flakiness dashboard should generate JSON in the background process
+        https://bugs.webkit.org/show_bug.cgi?id=123797
+
+        Reviewed by Alexey Proskuryakov.
+
+        * config.json: Add defaultBuildWaitInterval used by process-builds.php.
+        * init-database.sql: Add is_process column to builds table so that we can track of "unprocessed" builds.
+
+        * public/admin/process-builds.php: Added.
+        (process_latest_five_builds): Update flakiness states and generate JSONs for the latest five builds.
+        We go backwards in the time so that we don't end up infinite looping over a single build that fails.
+        (main): Call processed_builds with an exponential back off.
+
+        * public/api/report.php:
+        (store_results): No need to return build or builder ids.
+        (main): Don't update flakiness states or generate JSONs.
+
+        * public/include/test-results.php:
+        (add_build): Take the slave name; it doesn't make any sense to create a build without it and later
+        updating it in store_test_results.
+        (store_test_results): Once new test results are added, explicitly mark this build as unprocessed.
+
 2013-11-04  Ryosuke Niwa  <rniwa@webkit.org>
 
         Build fix. Update is_flaky when it's NULL.

diff --git a/Websites/test-results/config.json b/Websites/test-results/config.json
@@ -1,6 +1,7 @@
 {
     "debug": true,
     "jsonCacheMaxAge": 600,
+    "defaultBuildWaitInterval": 10,
     "dataDirectory": "../data",
     "database": {
         "host": "localhost",

diff --git a/Websites/test-results/init-database.sql b/Websites/test-results/init-database.sql
@@ -29,6 +29,7 @@ CREATE TABLE builds (
     start_time timestamp,
     end_time timestamp,
     slave integer REFERENCES slaves ON DELETE CASCADE,
+    is_processed boolean,
     CONSTRAINT builder_and_build_number_must_be_unique UNIQUE(builder, number));
 CREATE INDEX build_builder_index ON builds(builder);
 CREATE INDEX build_slave_index ON builds(slave);

diff --git a/Websites/test-results/public/admin/process-builds.php b/Websites/test-results/public/admin/process-builds.php
@@ -0,0 +1,64 @@
+<?php
+
+require_once('../include/test-results.php');
+
+ignore_user_abort(true); 
+set_time_limit(0);
+
+function process_latest_five_builds($db) {
+    $build_rows = $db->query_and_fetch_all('SELECT id, builder FROM builds
+        WHERE start_time IS NOT NULL AND is_processed = FALSE ORDER BY end_time DESC LIMIT 5');
+    if (!$build_rows)
+        return FALSE;
+
+    foreach ($build_rows as $row) {
+        echo "Build {$row['id']} for builder {$row['builder']}:\n";
+        echo "    Updating flakiness...";
+        flush();
+
+        $start_time = microtime(true);
+        update_flakiness_after_inserting_build($db, $row['id']);
+        $time = microtime(true) - $start_time;
+
+        echo "($time s)\n";
+        echo "    Generating JSONs...";
+        flush();
+
+        $start_time = microtime(true);
+        $generator = new ResultsJSONGenerator($db, $row['builder']);
+        $generator->generate('wrongexpectations');
+        $generator->generate('flaky');
+        $time = microtime(true) - $start_time;
+
+        echo "($time s)\n";
+        flush();
+
+        $db->query_and_get_affected_rows('UPDATE builds SET is_processed = TRUE where id = $1', array($row['id']));
+
+        sleep(1);
+    }
+
+    return TRUE;
+}
+
+function main() {
+    $db = new Database;
+    if (!$db->connect()) {
+        echo "Failed to connect to the database";
+        exit(1);
+    }
+
+    $wait = config('defaultBuildWaitInterval');
+    while (1) {
+        if (process_latest_five_builds($db))
+            $wait = max(1, $wait * 0.8);
+        else
+            $wait *= 2;
+        echo "Sleeping $wait s...\n";
+        sleep($wait);
+    }
+}
+
+main();
+
+?>
diff --git a/Websites/test-results/public/api/report.php b/Websites/test-results/public/api/report.php
@@ -12,7 +12,8 @@ function store_results($db, $master, $builder_name, $build_number, $start_time,
     if (!$builder_id)
         exit_with_error('FailedToInsertBuilder', array('master' => $master, 'builderName' => $builder_name));
 
-    $build_id = add_build($db, $builder_id, $build_number);
+    $slave_id = add_slave($db, $_POST['build_slave']);
+    $build_id = add_build($db, $builder_id, $build_number, $slave_id);
     if (!$build_id)
         exit_with_error('FailedToInsertBuild', array('builderId' => $builder_id, 'buildNumber' => $build_number));
 
@@ -30,11 +31,8 @@ function store_results($db, $master, $builder_name, $build_number, $start_time,
             or exit_with_error('FailedToInsertRevision', array('name' => $repository_name, 'data' => $revision_data));
     }
 
-    $slave_id = add_slave($db, $_POST['build_slave']);
-    if (!store_test_results($db, $test_results, $build_id, $start_time, $end_time, $slave_id))
+    if (!store_test_results($db, $test_results, $build_id, $start_time, $end_time))
         exit_with_error('FailedToStoreResults', array('buildId' => $build_id));
-
-    return array('build_id' => $build_id, 'builder_id' => $builder_id);
 }
 
 function main() {
@@ -66,26 +64,8 @@ function main() {
     $json_path = $_FILES['file']['tmp_name'];
 
     $db = connect();
-    $builder_and_build = store_results($db, $master, $builder_name, $build_number, $start_time, $end_time, $revisions, $json_path);
-    @ob_end_clean();
-    ignore_user_abort();
-    ob_start();
-
+    store_results($db, $master, $builder_name, $build_number, $start_time, $end_time, $revisions, $json_path);
     echo_success();
-
-    header('Connection: close');
-    header('Content-Length: ' . ob_get_length());
-
-    @ob_end_flush();
-    flush();
-    if (function_exists('fastcgi_finish_request'))
-        fastcgi_finish_request();
-
-    update_flakiness_after_inserting_build($db, $builder_and_build['build_id']);
-
-    $generator = new ResultsJSONGenerator($db, $builder_and_build['builder_id']);
-    $generator->generate('wrongexpectations');
-    $generator->generate('flaky');
 }
 
 main();

diff --git a/Websites/test-results/public/include/test-results.php b/Websites/test-results/public/include/test-results.php
@@ -15,8 +15,8 @@ function add_builder($db, $master, $builder_name) {
     return $db->select_or_insert_row('builders', NULL, array('master' => $master, 'name' => $builder_name));
 }
 
-function add_build($db, $builder_id, $build_number) {
-    return $db->select_or_insert_row('builds', NULL, array('builder' => $builder_id, 'number' => $build_number));
+function add_build($db, $builder_id, $build_number, $slave_id) {
+    return $db->select_or_insert_row('builds', NULL, array('builder' => $builder_id, 'number' => $build_number, 'slave' => $slave_id));
 }
 
 function add_slave($db, $name) {
@@ -34,15 +34,15 @@ function fetch_and_parse_test_results_json($url, $jsonp = FALSE) {
     return json_decode($json_contents, true);
 }
 
-function store_test_results($db, $test_results, $build_id, $start_time, $end_time, $slave_id) {
+function store_test_results($db, $test_results, $build_id, $start_time, $end_time) {
     $db->begin_transaction();
 
     try {
         recursively_add_test_results($db, $build_id, $test_results['tests'], '');
 
         $db->query_and_get_affected_rows(
-            'UPDATE builds SET (start_time, end_time, slave) = (least($1, start_time), greatest($2, end_time), $3) WHERE id = $4',
-            array($start_time->format('Y-m-d H:i:s.u'), $end_time->format('Y-m-d H:i:s.u'), $slave_id, $build_id));
+            'UPDATE builds SET (start_time, end_time, is_processed) = (least($1, start_time), greatest($2, end_time), FALSE) WHERE id = $3',
+            array($start_time->format('Y-m-d H:i:s.u'), $end_time->format('Y-m-d H:i:s.u'), $build_id));
         $db->commit_transaction();
     } catch (Exception $e) {
         $db->rollback_transaction();