Run wdspec tests in a different job

servo · Jun 14, 2019 · 2e0556c · 2e0556c
1 parent 5f4074a
commit 2e0556c
Show file tree

Hide file tree

Showing 134 changed files with 1,279 additions and 21 deletions.
diff --git a/etc/ci/buildbot_steps.yml b/etc/ci/buildbot_steps.yml
@@ -57,6 +57,7 @@ linux-rel-wpt:
     - ./mach test-wpt --release --binary-arg=--multiprocess --processes 24 --log-raw test-wpt-mp.log --log-errorsummary wpt-mp-errorsummary.log eventsource
     - ./mach test-wpt --release --product=servodriver --headless tests/wpt/mozilla/tests/mozilla/DOMParser.html tests/wpt/mozilla/tests/css/per_glyph_font_fallback_a.html tests/wpt/mozilla/tests/css/img_simple.html tests/wpt/mozilla/tests/mozilla/secure.https.html
     - ./mach test-wpt --release --product=servodriver --headless --log-raw test-bluetooth.log --log-errorsummary bluetooth-errorsummary.log bluetooth
+    - ./mach test-wpt --release --headless --timeout-multiplier=4 --log-raw test-wdspec.log --log-errorsummary wdspec-errorsummary.log webdriver
     - ./etc/ci/clean_build_artifacts.sh
 
 linux-rel-css:

diff --git a/etc/ci/update-wpt-checkout b/etc/ci/update-wpt-checkout
@@ -11,6 +11,7 @@ set -o pipefail
 REMOTE_NAME=sync-fork
 LOG_FILE=test-wpt.log
 BLUETOOTH_LOG_FILE=test-wpt-bluetooth.log
+WDSPEC_LOG_FILE=test-wpt-wdspec.log
 CURRENT_DATE=$(date +"%d-%m-%Y")
 BRANCH_NAME="wpt_update"
 REMOTE_BRANCH_NAME="wpt_update_${CURRENT_DATE}"
@@ -69,12 +70,16 @@ function unsafe_run_tests() {
     # Run the bluetooth testsuite, which uses the webdriver test harness.
     ./mach test-wpt --release --product=servodriver --headless \
            --log-raw "${2}" /bluetooth --always-succeed || return 2
+
+    # Run the wdspec testsuite
+    ./mach test-wpt --release --timeout-multiplier=4 --log-raw "${3}" /webdriver \
+           --always-succeed || return 3
 }
 
 # Using an existing log file, update the expected test results and amend the
 # last commit with the new results.
 function unsafe_update_metadata() {
-    ./mach update-wpt "${1}" "${2}" || return 1
+    ./mach update-wpt "${1}" "${2}" "${3}" || return 1
     # Ensure any new directories or ini files are included in these changes.
     git add tests/wpt/metadata tests/wpt/mozilla/meta || return 2
     # Merge all changes with the existing commit.
@@ -139,11 +144,11 @@ function pull_from_upstream() {
 }
 
 function run_tests() {
-    unsafe_run_tests "${1}" "${2}" || { code="${?}"; cleanup; return "${code}"; }
+    unsafe_run_tests "${1}" "${2}" "${3}" || { code="${?}"; cleanup; return "${code}"; }
 }
 
 function update_metadata() {
-    unsafe_update_metadata "${1}" "${2}" || { code="${?}"; cleanup; return "${code}"; }
+    unsafe_update_metadata "${1}" "${2}" "${3}" || { code="${?}"; cleanup; return "${code}"; }
 }
 
 function open_pull_request() {
@@ -153,8 +158,8 @@ function open_pull_request() {
 SCRIPT_NAME="${0}"
 
 function update_test_results() {
-    run_tests "${LOG_FILE}" "${BLUETOOTH_LOG_FILE}"
-    update_metadata "${LOG_FILE}" "${BLUETOOTH_LOG_FILE}"
+    run_tests "${LOG_FILE}" "${BLUETOOTH_LOG_FILE}" "${WDSPEC_LOG_FILE}"
+    update_metadata "${LOG_FILE}" "${BLUETOOTH_LOG_FILE}" "${WDSPEC_LOG_FILE}"
 }
 
 function fetch_upstream_changes() {

diff --git a/etc/taskcluster/decision_task.py b/etc/taskcluster/decision_task.py
@@ -613,6 +613,11 @@ def wpt_chunks(platform, make_chunk_task, build_task, total_chunks, processes,
                     --log-errorsummary bluetooth-errorsummary.log \
                     bluetooth \
                     | cat
+                time ./mach test-wpt --release --processes $PROCESSES --timeout-multiplier=4 \
+                    --headless --log-raw test-wdspec.log \
+                    --log-errorsummary wdspec-errorsummary.log \
+                    webdriver \
+                    | cat
             """)
         # `test-wpt` is piped into `cat` so that stdout is not a TTY
         # and wptrunner does not use "interactive mode" formatting:

diff --git a/tests/wpt/include.ini b/tests/wpt/include.ini
@@ -125,20 +125,6 @@ skip: true
   skip: false
 [WebCryptoAPI]
   skip: false
-[webdriver]
-  skip: true
-  [tests]
-    skip: true
-    [add_cookie]
-      skip: false
-    [delete_all_cookies]
-      skip: false
-    [delete_cookie]
-      skip: false
-    [get_named_cookie]
-      skip: false
-    [new_session]
-      skip: false
 [webgl]
   skip: false
 [webvr]

diff --git a/tests/wpt/metadata/webdriver/tests/accept_alert/accept.py.ini b/tests/wpt/metadata/webdriver/tests/accept_alert/accept.py.ini
@@ -0,0 +1,2 @@
+[accept.py]
+  disabled: Unimplemented WebDriver command
diff --git a/tests/wpt/metadata/webdriver/tests/back/back.py.ini b/tests/wpt/metadata/webdriver/tests/back/back.py.ini
@@ -0,0 +1,22 @@
+[back.py]
+  [test_history_pushstate]
+    expected: ERROR
+
+  [test_no_browsing_history]
+    expected: ERROR
+
+  [test_null_response_value]
+    expected: ERROR
+
+  [test_fragments]
+    expected: ERROR
+
+  [test_data_urls]
+    expected: ERROR
+
+  [test_no_browsing_context]
+    expected: ERROR
+
+  [test_dismissed_beforeunload]
+    expected: ERROR
+
diff --git a/tests/wpt/metadata/webdriver/tests/back/user_prompts.py.ini b/tests/wpt/metadata/webdriver/tests/back/user_prompts.py.ini
@@ -0,0 +1,2 @@
+[user_prompts.py]
+  disabled: User prompts not yet implemented
diff --git a/tests/wpt/metadata/webdriver/tests/close_window/close.py.ini b/tests/wpt/metadata/webdriver/tests/close_window/close.py.ini
@@ -0,0 +1,2 @@
+[close.py]
+  disabled: Unimplemented WebDriver command
diff --git a/tests/wpt/metadata/webdriver/tests/close_window/user_prompts.py.ini b/tests/wpt/metadata/webdriver/tests/close_window/user_prompts.py.ini
@@ -0,0 +1,2 @@
+[user_prompts.py]
+  disabled: User prompts not yet implemented
diff --git a/tests/wpt/metadata/webdriver/tests/delete_session/delete.py.ini b/tests/wpt/metadata/webdriver/tests/delete_session/delete.py.ini
@@ -0,0 +1,7 @@
+[delete.py]
+  [test_null_response_value]
+    expected: FAIL
+
+  [test_dismissed_beforeunload_prompt]
+    expected: FAIL
+
diff --git a/tests/wpt/metadata/webdriver/tests/dismiss_alert/dismiss.py.ini b/tests/wpt/metadata/webdriver/tests/dismiss_alert/dismiss.py.ini
@@ -0,0 +1,2 @@
+[dismiss.py]
+  disabled: Unimplemented WebDriver command
diff --git a/tests/wpt/metadata/webdriver/tests/element_clear/clear.py.ini b/tests/wpt/metadata/webdriver/tests/element_clear/clear.py.ini
@@ -0,0 +1,2 @@
+[clear.py]
+  disabled: Unimplemented WebDriver command
diff --git a/tests/wpt/metadata/webdriver/tests/element_clear/user_prompts.py.ini b/tests/wpt/metadata/webdriver/tests/element_clear/user_prompts.py.ini
@@ -0,0 +1,2 @@
+[user_prompts.py]
+  disabled: User prompts not yet implemented
diff --git a/tests/wpt/metadata/webdriver/tests/element_click/bubbling.py.ini b/tests/wpt/metadata/webdriver/tests/element_click/bubbling.py.ini
@@ -0,0 +1,2 @@
+[bubbling.py]
+  disabled: Unimplemented WebDriver command
diff --git a/tests/wpt/metadata/webdriver/tests/element_click/center_point.py.ini b/tests/wpt/metadata/webdriver/tests/element_click/center_point.py.ini
@@ -0,0 +1,2 @@
+[center_point.py]
+  disabled: Unimplemented WebDriver command
diff --git a/tests/wpt/metadata/webdriver/tests/element_click/click.py.ini b/tests/wpt/metadata/webdriver/tests/element_click/click.py.ini
@@ -0,0 +1,2 @@
+[click.py]
+  disabled: Unimplemented WebDriver command
diff --git a/tests/wpt/metadata/webdriver/tests/element_click/file_upload.py.ini b/tests/wpt/metadata/webdriver/tests/element_click/file_upload.py.ini
@@ -0,0 +1,2 @@
+[file_upload.py]
+  disabled: Unimplemented WebDriver command
diff --git a/tests/wpt/metadata/webdriver/tests/element_click/interactability.py.ini b/tests/wpt/metadata/webdriver/tests/element_click/interactability.py.ini
@@ -0,0 +1,2 @@
+[interactability.py]
+  disabled: Unimplemented WebDriver command
diff --git a/tests/wpt/metadata/webdriver/tests/element_click/navigate.py.ini b/tests/wpt/metadata/webdriver/tests/element_click/navigate.py.ini
@@ -0,0 +1,2 @@
+[navigate.py]
+  disabled: Unimplemented WebDriver command
diff --git a/tests/wpt/metadata/webdriver/tests/element_click/scroll_into_view.py.ini b/tests/wpt/metadata/webdriver/tests/element_click/scroll_into_view.py.ini
@@ -0,0 +1,2 @@
+[scroll_into_view.py]
+  disabled: Unimplemented WebDriver command
diff --git a/tests/wpt/metadata/webdriver/tests/element_click/select.py.ini b/tests/wpt/metadata/webdriver/tests/element_click/select.py.ini
@@ -0,0 +1,2 @@
+[select.py]
+  disabled: Unimplemented WebDriver command
diff --git a/tests/wpt/metadata/webdriver/tests/element_click/stale.py.ini b/tests/wpt/metadata/webdriver/tests/element_click/stale.py.ini
@@ -0,0 +1,2 @@
+[stale.py]
+  disabled: Unimplemented WebDriver command
diff --git a/tests/wpt/metadata/webdriver/tests/element_click/user_prompts.py.ini b/tests/wpt/metadata/webdriver/tests/element_click/user_prompts.py.ini
@@ -0,0 +1,2 @@
+[user_prompts.py]
+  disabled: User prompts not yet implemented
diff --git a/tests/wpt/metadata/webdriver/tests/element_send_keys/content_editable.py.ini b/tests/wpt/metadata/webdriver/tests/element_send_keys/content_editable.py.ini
@@ -0,0 +1,7 @@
+[content_editable.py]
+  [test_sets_insertion_point_to_after_last_text_node]
+    expected: FAIL
+
+  [test_sets_insertion_point_to_end]
+    expected: FAIL
+
diff --git a/tests/wpt/metadata/webdriver/tests/element_send_keys/events.py.ini b/tests/wpt/metadata/webdriver/tests/element_send_keys/events.py.ini
@@ -0,0 +1,16 @@
+[events.py]
+  [test_form_control_send_text[input\]]
+    expected: FAIL
+
+  [test_form_control_send_text[textarea\]]
+    expected: FAIL
+
+  [test_not_blurred[input\]]
+    expected: FAIL
+
+  [test_file_upload]
+    expected: FAIL
+
+  [test_not_blurred[textarea\]]
+    expected: FAIL
+
diff --git a/tests/wpt/metadata/webdriver/tests/element_send_keys/file_upload.py.ini b/tests/wpt/metadata/webdriver/tests/element_send_keys/file_upload.py.ini
@@ -0,0 +1,49 @@
+[file_upload.py]
+  [test_multiple_files]
+    expected: FAIL
+
+  [test_multiple_files_last_path_not_found]
+    expected: FAIL
+
+  [test_multiple_files_without_multiple_attribute]
+    expected: FAIL
+
+  [test_obscured]
+    expected: FAIL
+
+  [test_multiple_files_send_twice]
+    expected: FAIL
+
+  [test_single_file_replaces_without_multiple_attribute]
+    expected: FAIL
+
+  [test_strict_display_none[capabilities0\]]
+    expected: ERROR
+
+  [test_outside_viewport]
+    expected: FAIL
+
+  [test_multiple_files_reset_with_element_clear]
+    expected: FAIL
+
+  [test_strict_hidden[capabilities0\]]
+    expected: ERROR
+
+  [test_single_file_appends_with_multiple_attribute]
+    expected: FAIL
+
+  [test_empty_text]
+    expected: FAIL
+
+  [test_single_file]
+    expected: FAIL
+
+  [test_display_none]
+    expected: FAIL
+
+  [test_hidden]
+    expected: FAIL
+
+  [test_transparent]
+    expected: FAIL
+
diff --git a/tests/wpt/metadata/webdriver/tests/element_send_keys/form_controls.py.ini b/tests/wpt/metadata/webdriver/tests/element_send_keys/form_controls.py.ini
@@ -0,0 +1,13 @@
+[form_controls.py]
+  [test_textarea]
+    expected: FAIL
+
+  [test_input_append]
+    expected: FAIL
+
+  [test_textarea_append]
+    expected: FAIL
+
+  [test_input]
+    expected: FAIL
+
diff --git a/tests/wpt/metadata/webdriver/tests/element_send_keys/interactability.py.ini b/tests/wpt/metadata/webdriver/tests/element_send_keys/interactability.py.ini
@@ -0,0 +1,34 @@
+[interactability.py]
+  [test_disabled]
+    expected: FAIL
+
+  [test_not_a_focusable_element]
+    expected: FAIL
+
+  [test_visibility_hidden]
+    expected: FAIL
+
+  [test_transparent_element]
+    expected: FAIL
+
+  [test_iframe_is_interactable]
+    expected: FAIL
+
+  [test_readonly_element]
+    expected: FAIL
+
+  [test_obscured_element]
+    expected: FAIL
+
+  [test_document_element_is_interactable]
+    expected: FAIL
+
+  [test_body_is_interactable]
+    expected: FAIL
+
+  [test_hidden]
+    expected: FAIL
+
+  [test_display_none]
+    expected: FAIL
+
diff --git a/tests/wpt/metadata/webdriver/tests/element_send_keys/scroll_into_view.py.ini b/tests/wpt/metadata/webdriver/tests/element_send_keys/scroll_into_view.py.ini
@@ -0,0 +1,16 @@
+[scroll_into_view.py]
+  [test_element_outside_of_not_scrollable_viewport]
+    expected: FAIL
+
+  [test_option_select_container_outside_of_scrollable_viewport]
+    expected: FAIL
+
+  [test_contenteditable_element_outside_of_scrollable_viewport]
+    expected: FAIL
+
+  [test_element_outside_of_scrollable_viewport]
+    expected: FAIL
+
+  [test_option_stays_outside_of_scrollable_viewport]
+    expected: FAIL
+
diff --git a/tests/wpt/metadata/webdriver/tests/element_send_keys/send_keys.py.ini b/tests/wpt/metadata/webdriver/tests/element_send_keys/send_keys.py.ini
@@ -0,0 +1,28 @@
+[send_keys.py]
+  [test_invalid_text_type[value4\]]
+    expected: FAIL
+
+  [test_invalid_text_type[None\]]
+    expected: FAIL
+
+  [test_null_response_value]
+    expected: FAIL
+
+  [test_invalid_text_type[value3\]]
+    expected: FAIL
+
+  [test_invalid_text_type[1\]]
+    expected: FAIL
+
+  [test_null_parameter_value]
+    expected: FAIL
+
+  [test_stale_element]
+    expected: FAIL
+
+  [test_no_browsing_context]
+    expected: ERROR
+
+  [test_invalid_text_type[True\]]
+    expected: FAIL
+
diff --git a/tests/wpt/metadata/webdriver/tests/element_send_keys/user_prompts.py.ini b/tests/wpt/metadata/webdriver/tests/element_send_keys/user_prompts.py.ini
@@ -0,0 +1,2 @@
+[user_prompts.py]
+  disabled: User prompts not yet implemented
diff --git a/tests/wpt/metadata/webdriver/tests/execute_async_script/collections.py.ini b/tests/wpt/metadata/webdriver/tests/execute_async_script/collections.py.ini
@@ -0,0 +1,2 @@
+[collections.py]
+  disabled: unstable
diff --git a/tests/wpt/metadata/webdriver/tests/execute_async_script/execute_async.py.ini b/tests/wpt/metadata/webdriver/tests/execute_async_script/execute_async.py.ini
@@ -0,0 +1,2 @@
+[execute_async.py]
+  disabled: unstable
diff --git a/tests/wpt/metadata/webdriver/tests/execute_async_script/promise.py.ini b/tests/wpt/metadata/webdriver/tests/execute_async_script/promise.py.ini
@@ -0,0 +1,2 @@
+[promise.py]
+  disabled: unstable
diff --git a/tests/wpt/metadata/webdriver/tests/execute_async_script/properties.py.ini b/tests/wpt/metadata/webdriver/tests/execute_async_script/properties.py.ini
@@ -0,0 +1,2 @@
+[properties.py]
+  disabled: unstable
diff --git a/tests/wpt/metadata/webdriver/tests/execute_async_script/user_prompts.py.ini b/tests/wpt/metadata/webdriver/tests/execute_async_script/user_prompts.py.ini
@@ -0,0 +1,2 @@
+[user_prompts.py]
+  disabled: User prompts not yet implemented
diff --git a/tests/wpt/metadata/webdriver/tests/execute_script/collections.py.ini b/tests/wpt/metadata/webdriver/tests/execute_script/collections.py.ini
@@ -0,0 +1,25 @@
+[collections.py]
+  [test_html_form_controls_collection]
+    expected: FAIL
+
+  [test_html_options_collection]
+    expected: FAIL
+
+  [test_file_list]
+    expected: FAIL
+
+  [test_arguments]
+    expected: FAIL
+
+  [test_html_collection]
+    expected: FAIL
+
+  [test_array]
+    expected: FAIL
+
+  [test_node_list]
+    expected: FAIL
+
+  [test_html_all_collection]
+    expected: FAIL
+