forked from apache/druid
-
Notifications
You must be signed in to change notification settings - Fork 0
/
check_test_suite.py
executable file
·172 lines (140 loc) · 6.86 KB
/
check_test_suite.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
#!/usr/bin/env python3
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import subprocess
import sys
# this script does some primitive examination of git diff to determine if a test suite needs to be run or not
# these jobs should always be run, no matter what
always_run_jobs = ['license checks', '(openjdk8) packaging check', '(openjdk11) packaging check']
# ignore changes to these files completely since they don't impact CI, if the changes are only to these files then all
# of CI can be skipped. however, jobs which are always run will still be run even if only these files are changed
ignore_prefixes = ['.github', '.idea', '.asf.yaml', '.backportrc.json', '.codecov.yml', '.dockerignore', '.gitignore',
'.lgtm.yml', 'CONTRIBUTING.md', 'setup-hooks.sh', 'upload.sh', 'dev', 'distribution/docker',
'distribution/asf-release-process-guide.md', '.travis.yml', 'check_test_suite.py',
'check_test_suite_test.py', 'owasp-dependency-check-suppressions.xml']
# these files are docs changes
# if changes are limited to this set then we can skip web-console and java
# if no changes in this set we can skip docs
docs_prefixes = ['docs/', 'website/']
# travis docs job name
docs_jobs = ['docs']
# these files are web-console changes
# if changes are limited to this set then we can skip docs and java
# if no changes in this set we can skip web-console
web_console_prefixes = ['web-console/']
# travis web-console job name
web_console_jobs = ['web console', 'web console end-to-end test']
def check_ignore(file):
is_always_ignore = True in (file.startswith(prefix) for prefix in ignore_prefixes)
if is_always_ignore:
print("found ignorable file change: {}".format(file))
return is_always_ignore
def check_docs(file):
is_docs = True in (file.startswith(prefix) for prefix in docs_prefixes)
if is_docs:
print("found docs file change: {}".format(file))
return is_docs
def check_console(file):
is_console = True in (file.startswith(prefix) for prefix in web_console_prefixes)
if is_console:
print("found web-console file change: {}".format(file))
return is_console
def check_should_run_suite(suite, diff_files):
"""
try to determine if a test suite should run or not given a set files in a diff
:param suite: travis job name
:param diff_files: files changed in git diff
:return: True if the supplied suite needs to run, False otherwise
"""
if suite in always_run_jobs:
# you gotta do what you gotta do
return True
all_ignore = True
any_docs = False
all_docs = True
any_console = False
all_console = True
any_java = False
# go over all of the files in the diff and collect some information about the diff contents, we'll use this later
# to decide whether or not to run the suite
for f in diff_files:
is_ignore = check_ignore(f)
all_ignore = all_ignore and is_ignore
is_docs = check_docs(f)
any_docs = any_docs or is_docs
all_docs = all_docs and is_docs
is_console = check_console(f)
any_console = any_console or is_console
all_console = any_console and is_console
any_java = any_java or (not is_ignore and not is_docs and not is_console)
# if everything is ignorable, we can skip this suite
if all_ignore:
return False
# if the test suite is a doc job, return true if any of the files changed were docs
if suite in docs_jobs:
return any_docs
# if all of the changes are docs paths, but the current suite is not a docs job, we can skip
if all_docs:
return False
# if the test suite is a web console job, return true if any of the changes are web console files
if suite in web_console_jobs:
return any_console
# if all of the changes are web console paths, but the current suite is not a web console job, we can skip
if all_console:
return False
# if all of the files belong to known non-java groups, we can also skip java
# note that this should probably be reworked to much more selectively run the java jobs depending on the diff
if not any_java:
return False
# we don't know we can skip for sure, so lets run it
return True
def failWithUsage():
sys.stderr.write("usage: check_test_suite.py <test-suite-name>\n")
sys.stderr.write(" e.g., check_test_suite.py docs")
sys.exit(1)
if __name__ == '__main__':
suite_name = ""
# when run by travis, we run this script without arguments, so collect the test suite name from environment
# variables. if it doesn't exist, fail
if len(sys.argv) == 1:
if 'TRAVIS_JOB_NAME' in os.environ:
suite_name = os.environ['TRAVIS_JOB_NAME']
else:
failWithUsage()
elif len(sys.argv) == 2:
# to help with testing, can explicitly pass a test suite name
suite_name = sys.argv[1]
else:
failWithUsage()
# we only selectively run CI for PR builds, branch builds such as master and releases will always run all suites
is_pr = False
if 'TRAVIS_PULL_REQUEST' in os.environ and os.environ['TRAVIS_PULL_REQUEST'] != 'false':
is_pr = True
if not is_pr:
print("Not a pull request build, need to run all test suites")
sys.exit(1)
# this looks like it only gets the last commit, but the way travis PR builds work this actually gets the complete
# diff (since all commits from the PR are rebased onto the target branch and added as a single commit)
all_changed_files_string = subprocess.check_output("git diff --name-only HEAD~1", shell=True).decode('UTF-8')
all_changed_files = all_changed_files_string.splitlines()
print("Checking if suite '{}' needs to run test on diff:\n{}".format(suite_name, all_changed_files_string))
# we should always run all test suites for builds that are not for a pull request
needs_run = check_should_run_suite(suite_name, all_changed_files)
if needs_run:
print("Changes detected, need to run test suite '{}'".format(suite_name))
sys.exit(1)
print("No applicable changes detected, can skip test suite '{}'".format(suite_name))
sys.exit(0)