Skip to content

Commit 853ee2f

Browse files
naafiyanMongoDB Bot
naafiyan
authored and
MongoDB Bot
committed
SERVER-97276: Added evergreen and resmoke integration for running query-correctness-tests using query_tester (#29465)
GitOrigin-RevId: 6e11f11
1 parent c00baea commit 853ee2f

File tree

11 files changed

+234
-2
lines changed

11 files changed

+234
-2
lines changed

‎.github/CODEOWNERS

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,9 @@ WORKSPACE.bazel @10gen/devprod-build @svc-auto-approve-bot
138138
/buildscripts/resmokelib/testing/hooks/**/enable_change_stream.py @10gen/query-execution-change-streams @svc-auto-approve-bot
139139
/buildscripts/resmokelib/testing/hooks/**/run_query_stats.py @10gen/query-integration-query-stats @svc-auto-approve-bot
140140

141+
# The following patterns are parsed from ./buildscripts/resmokelib/testing/testcases/OWNERS.yml
142+
/buildscripts/resmokelib/testing/testcases/**/query_tester_server_test.py @10gen/query-optimization @svc-auto-approve-bot
143+
141144
# The following patterns are parsed from ./buildscripts/smoke_tests/OWNERS.yml
142145
/buildscripts/smoke_tests/**/server_programmability.yml @10gen/server-programmability @svc-auto-approve-bot
143146

‎buildscripts/resmokeconfig/suites/query_tester_custom_test.yml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
test_kind: query_tester_server_test
2+
description: |
3+
This suite will run custom tests on the mongo server using QueryTester's mongotest binary.
4+
5+
selector:
6+
roots:
7+
- src/mongo/db/query/query_tester/tests/query-correctness-tests/custom_tests/*/
8+
executor:
9+
hooks:
10+
- class: CleanEveryN
11+
n: 1
12+
fixture:
13+
class: MongoDFixture

‎buildscripts/resmokeconfig/suites/query_tester_generated_test.yml

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
test_kind: query_tester_server_test
2+
description: |
3+
This suite will run generated tests on the mongo server using QueryTester's mongotest binary.
4+
5+
selector:
6+
roots:
7+
- src/mongo/db/query/query_tester/tests/*/generated_tests/*/
8+
exclude_files:
9+
- src/mongo/db/query/query_tester/tests/selfTests/
10+
executor:
11+
hooks:
12+
- class: CleanEveryN
13+
n: 1
14+
fixture:
15+
class: MongoDFixture

‎buildscripts/resmokelib/selector.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -907,6 +907,7 @@ def __init__(self, roots, include_files=None, exclude_files=None):
907907
"bulk_write_cluster_js_test": (_JSTestSelectorConfig, _JSTestSelector),
908908
"magic_restore_js_test": (_JSTestSelectorConfig, _JSTestSelector),
909909
"query_tester_self_test": (_FileBasedSelectorConfig, _Selector),
910+
"query_tester_server_test": (_FileBasedSelectorConfig, _Selector),
910911
}
911912

912913

‎buildscripts/resmokelib/testing/testcases/OWNERS.yml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
version: 1.0.0
2+
filters:
3+
- "query_tester_server_test.py":
4+
approvers:
5+
- 10gen/query-optimization

‎buildscripts/resmokelib/testing/testcases/query_tester_server_test.py

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
"""The unittest.TestCase for QueryTester server correctness tests."""
2+
3+
import os
4+
5+
from buildscripts.resmokelib import config as _config
6+
from buildscripts.resmokelib import core, logging
7+
from buildscripts.resmokelib.testing.testcases import interface
8+
9+
10+
class QueryTesterServerTestCase(interface.ProcessTestCase):
11+
"""A QueryTester server test to execute."""
12+
13+
REGISTERED_NAME = "query_tester_server_test"
14+
15+
def __init__(self, logger: logging.Logger, test_dir: list[str]):
16+
"""Initialize QueryTesterServerTestCase.
17+
test_dir must contain one dir that contains .test files, their corresponding .results and a .coll file
18+
To run multiple test dirs, you would create an instance of QueryTesterServerTestCase for each one.
19+
"""
20+
assert len(test_dir) == 1
21+
interface.ProcessTestCase.__init__(self, logger, "QueryTesterServerTest", test_dir[0])
22+
self.test_files = [
23+
os.path.join(test_dir[0], f) for f in os.listdir(test_dir[0]) if f.endswith(".test")
24+
]
25+
26+
def _make_process(self):
27+
return core.programs.generic_program(
28+
self.logger,
29+
[
30+
_config.DEFAULT_MONGOTEST_EXECUTABLE,
31+
"--uri",
32+
self.fixture.get_internal_connection_string(),
33+
*[cmd for f in self.test_files for cmd in ("-t", f)],
34+
"--drop",
35+
"--load",
36+
"--mode",
37+
"compare",
38+
"-v",
39+
"--diff",
40+
"plain",
41+
],
42+
)

‎etc/evergreen_yml_components/definitions.yml

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -145,8 +145,9 @@ functions:
145145
args:
146146
- "-c"
147147
- |
148-
if [ -z "${gcov_tool}" ]; then
149-
echo "Not a code coverage repo, skipping git clone."
148+
# clone_repo_for_test indicates that we want to clone the mongo git repo.
149+
if [ -z "${gcov_tool}" ] && [ "${clone_repo_for_test}" != "true" ]; then
150+
echo "Not a code coverage repo or task that requires a git checkout, skipping git clone."
150151
exit 0
151152
fi
152153
@@ -1780,6 +1781,26 @@ functions:
17801781
include_expansions_in_env:
17811782
- github_token
17821783

1784+
"setup query_tester test repos":
1785+
- *f_expansions_write
1786+
- command: github.generate_token
1787+
params:
1788+
owner: 10gen
1789+
repo: query-correctness-tests
1790+
expansion_name: github_token
1791+
permissions:
1792+
metadata: read
1793+
contents: read
1794+
- command: subprocess.exec
1795+
display_name: "query_tester test repos setup sh"
1796+
params:
1797+
binary: bash
1798+
args:
1799+
- "./src/evergreen/query_tester_test_repo_setup.sh"
1800+
include_expansions_in_env:
1801+
- github_token
1802+
- repo_name
1803+
17831804
"lint fuzzer sanity patch":
17841805
- *f_expansions_write
17851806
- command: subprocess.exec

‎etc/evergreen_yml_components/tasks/resmoke/server_divisions/query/tasks.yml

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1742,6 +1742,66 @@ tasks:
17421742
vars:
17431743
suite: query_tester_self_test
17441744

1745+
- <<: *task_template
1746+
name: query_tester_generated_test_1
1747+
patch_only: true # TODO: SERVER-98733 - Remove to run on waterfall.
1748+
tags: ["assigned_to_jira_team_server_query_optimization", "default"]
1749+
commands:
1750+
- func: "do setup"
1751+
vars:
1752+
# Set this variable to true so that we clone the mongo git repo, including the
1753+
# .gitattributes file that provides hunk header overrides when running git diff in the
1754+
# query_tester_generated_test task. This will trigger the "conditionally clone repo"
1755+
# function during "do setup."
1756+
clone_repo_for_test: true
1757+
- func: "setup query_tester test repos"
1758+
vars:
1759+
# The name of test repo that we want this task to clone.
1760+
repo_name: "query-correctness-tests"
1761+
- func: "run tests"
1762+
vars:
1763+
suite: query_tester_generated_test
1764+
1765+
- <<: *task_template
1766+
name: query_tester_generated_test_2
1767+
patch_only: true # TODO: SERVER-98733 - Remove to run on waterfall.
1768+
tags: ["assigned_to_jira_team_server_query_optimization", "default"]
1769+
commands:
1770+
- func: "do setup"
1771+
vars:
1772+
# Set this variable to true so that we clone the mongo git repo, including the
1773+
# .gitattributes file that provides hunk header overrides when running git diff in the
1774+
# query_tester_generated_test task. This will trigger the "conditionally clone repo"
1775+
# function during "do setup."
1776+
clone_repo_for_test: true
1777+
- func: "setup query_tester test repos"
1778+
vars:
1779+
# The name of test repo that we want this task to clone.
1780+
repo_name: "query-correctness-tests-1"
1781+
- func: "run tests"
1782+
vars:
1783+
suite: query_tester_generated_test
1784+
1785+
- <<: *task_template
1786+
name: query_tester_generated_test_3
1787+
patch_only: true # TODO: SERVER-98733 - Remove to run on waterfall.
1788+
tags: ["assigned_to_jira_team_server_query_optimization", "default"]
1789+
commands:
1790+
- func: "do setup"
1791+
vars:
1792+
# Set this variable to true so that we clone the mongo git repo, including the
1793+
# .gitattributes file that provides hunk header overrides when running git diff in the
1794+
# query_tester_generated_test task. This will trigger the "conditionally clone repo"
1795+
# function during "do setup."
1796+
clone_repo_for_test: true
1797+
- func: "setup query_tester test repos"
1798+
vars:
1799+
# The name of test repo that we want this task to clone.
1800+
repo_name: "query-correctness-tests-2"
1801+
- func: "run tests"
1802+
vars:
1803+
suite: query_tester_generated_test
1804+
17451805
################################################
17461806
# Query Integration tasks #
17471807
################################################

‎evergreen/query_tester_test_repo_setup.sh

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
# This script is designed to clone a specific GitHub repository that contains server tests used by query_tester.
2+
# It ensures the repository's state is set to a specific commit for consistent testing. This associates each server commit with a test repository commit.
3+
#
4+
# Inputs:
5+
# Environment Variables:
6+
# repo_name: The name of the repository to clone.
7+
# github_token: GitHub access token for authentication.
8+
# Configuration File:
9+
# test_repos.conf: A file containing a mapping of repository names to corresponding commit hashes in the format <repo_name>:<commit_hash>.
10+
# Outputs:
11+
# A cloned repository in src/mongo/db/query/query_tester/tests/<repo_name> that is reset to the appropriate commit hash.
12+
13+
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)"
14+
. "$DIR/prelude.sh"
15+
16+
set -o errexit
17+
set -o verbose
18+
19+
# Ensure that repo_name has been defined in evergreen tasks.
20+
if [ -z "${repo_name}" ]; then
21+
echo "Error: No repository name provided."
22+
exit 1
23+
fi
24+
25+
cd src/src/mongo/db/query/query_tester/tests
26+
27+
# Find the commit hash we want to use for this repo from the config file.
28+
commit_hash=$(grep "^${repo_name}:" test_repos.conf | awk -F ':' '{print $2}')
29+
30+
if [ -z "$commit_hash" ]; then
31+
echo "Error: Repository with name ${repo_name} not found in configuration file."
32+
exit 1
33+
fi
34+
35+
for i in {1..5}; do
36+
git clone https://x-access-token:${github_token}@github.com/10gen/${repo_name}.git ${repo_name} && RET=0 && break || RET=$? && sleep 5
37+
echo "Failed to clone github.com:10gen/${repo_name}.git, retrying..."
38+
done
39+
40+
if [ $RET -ne 0 ]; then
41+
echo "Failed to clone git@github.com:10gen/${repo_name}.git"
42+
exit $RET
43+
fi
44+
45+
cd ${repo_name} || exit 1
46+
47+
git reset --hard ${commit_hash}
48+
RESET_RET=$?
49+
50+
# Make sure the commit hash refers to a valid commit in the repo.
51+
if [ $RESET_RET -ne 0 ]; then
52+
echo "Failed to reset to commit ${commit_hash} in ${repo_name}."
53+
exit $RESET_RET
54+
else
55+
echo "Successfully reset ${repo_name} to commit ${commit_hash}."
56+
fi

‎src/mongo/db/query/query_tester/.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,5 @@
22
*.core
33
*.new
44
*/1727963727765/**/*.results
5+
tests/*
6+
!tests/selfTests/

0 commit comments

Comments
 (0)