Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

add CI visibility links to KMT run jobs #26948

Merged
merged 6 commits into from
Jun 21, 2024
Merged

add CI visibility links to KMT run jobs #26948

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
af7b5c7
add ci visibility links to KMT run jobs
brycekahle Jun 20, 2024
f27cf2d
add test run links
brycekahle Jun 20, 2024
20be2be
fix tests
brycekahle Jun 20, 2024
595e153
fix tests
brycekahle Jun 20, 2024
88eab2c
address review comments
brycekahle Jun 21, 2024
d1a725b
fix slash quote
brycekahle Jun 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .gitlab/kernel_matrix_testing/common.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,7 @@
variables:
AWS_EC2_SSH_KEY_FILE: $CI_PROJECT_DIR/ssh_key
RETRY: 2
EXTERNAL_LINKS_PATH: external_links_$CI_JOB_ID.json
before_script:
- export DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/aws_ssm_get_wrapper.sh $API_KEY_ORG2_SSM_NAME)
- !reference [.kmt_new_profile]
Expand All @@ -207,6 +208,7 @@
- echo "CI_JOB_ID=${CI_JOB_ID}" >> $DD_AGENT_TESTING_DIR/job_env.txt
- echo "CI_JOB_NAME=${CI_JOB_NAME}" >> $DD_AGENT_TESTING_DIR/job_env.txt
- echo "CI_JOB_STAGE=${CI_JOB_STAGE}" >> $DD_AGENT_TESTING_DIR/job_env.txt
- inv -e gitlab.generate-ci-visibility-links --output=$EXTERNAL_LINKS_PATH
script:
- INSTANCE_IP=$(jq --exit-status --arg ARCH $ARCH -r '.[$ARCH].ip' $CI_PROJECT_DIR/stack.output)
- !reference [.shared_filters_and_queries]
Expand All @@ -233,3 +235,6 @@
- $DD_AGENT_TESTING_DIR/junit-$ARCH-$TAG-$TEST_SET.tar.gz
- $DD_AGENT_TESTING_DIR/testjson-$ARCH-$TAG-$TEST_SET.tar.gz
- $CI_PROJECT_DIR/logs
reports:
annotations:
- $EXTERNAL_LINKS_PATH
50 changes: 45 additions & 5 deletions tasks/gitlab_helpers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,12 @@

from invoke import task

from tasks.libs.ciproviders.gitlab_api import BASE_URL as GITLAB_BASE_URL
from tasks.libs.common.color import Color, color_message

CI_VISIBILITY_URL = "https://app.datadoghq.com/ci/pipeline-executions"
CI_VISIBILITY_BASE_URL = "https://app.datadoghq.com/ci"
CI_VISIBILITY_URL = f"{CI_VISIBILITY_BASE_URL}/pipeline-executions"
TEST_VISIBILITY_URL = f"{CI_VISIBILITY_BASE_URL}/test-runs"


@task
Expand Down Expand Up @@ -55,20 +58,32 @@ def create_gitlab_annotations_report(ci_job_id: str, ci_job_name: str):
{
"external_link": {
"label": "CI Visibility: This job instance",
"url": get_link_to_job_id(ci_job_id),
"url": get_link_to_pipeline_job_id(ci_job_id),
}
},
{
"external_link": {
"label": "Test Visibility: This job test runs",
"url": get_link_to_test_runs(ci_job_id),
}
},
{
"external_link": {
"label": "CI Visibility: This job on main",
"url": get_link_to_job_on_main(ci_job_name),
"url": get_link_to_pipeline_job_on_main(ci_job_name),
}
},
{
"external_link": {
"label": "Test Visibility: This job test runs on main",
"url": get_link_to_test_runs_on_main(ci_job_name),
}
},
]
}


def get_link_to_job_id(job_id: str):
def get_link_to_pipeline_job_id(job_id: str):
query_params = {
"ci_level": "job",
"@ci.job.id": job_id,
Expand All @@ -79,7 +94,7 @@ def get_link_to_job_id(job_id: str):
return f"{CI_VISIBILITY_URL}?query={quoted_query_string}"


def get_link_to_job_on_main(job_name: str):
def get_link_to_pipeline_job_on_main(job_name: str):
# explicitly escape double quotes
job_name = job_name.replace('"', '\\"')
query_params = {
Expand All @@ -94,5 +109,30 @@ def get_link_to_job_on_main(job_name: str):
return f"{CI_VISIBILITY_URL}?query={quoted_query_string}"


def get_link_to_test_runs(job_id: str):
query_params = {
"test_level": "test",
"@ci.job.url": f"\"{GITLAB_BASE_URL}/DataDog/datadog-agent/-/jobs/{job_id}\"",
"@test.service": "datadog-agent",
}
query_string = to_query_string(query_params)
quoted_query_string = quote(string=query_string, safe="")
return f"{TEST_VISIBILITY_URL}?query={quoted_query_string}"


def get_link_to_test_runs_on_main(job_name: str):
job_name = job_name.replace('"', '\\"')
query_params = {
"test_level": "test",
# wrapping in double quotes
"@ci.job.name": f'"{job_name}"',
"@git.branch": "main",
"@test.service": "datadog-agent",
}
query_string = to_query_string(query_params)
quoted_query_string = quote(string=query_string, safe="")
return f"{TEST_VISIBILITY_URL}?query={quoted_query_string}"


def to_query_string(params: dict):
return " ".join(f"{k}:{v}" for k, v in params.items())
39 changes: 39 additions & 0 deletions tasks/unit-tests/gitlab_helpers_tests.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
import unittest
from urllib.parse import quote

from tasks.gitlab_helpers import (
CI_VISIBILITY_URL,
TEST_VISIBILITY_URL,
create_gitlab_annotations_report,
)
from tasks.libs.ciproviders.gitlab_api import BASE_URL as GITLAB_BASE_URL


class TestCreateGitlabAnnotations(unittest.TestCase):
Expand All @@ -19,12 +22,24 @@ def test_create_gitlab_annotations_report_basic_name(self):
"url": f"{CI_VISIBILITY_URL}?query=ci_level%3Ajob%20%40ci.job.id%3A123%20%40ci.pipeline.name%3ADataDog%2Fdatadog-agent",
}
},
{
"external_link": {
"label": "Test Visibility: This job test runs",
"url": f"{TEST_VISIBILITY_URL}?query=test_level%3Atest%20%40ci.job.url%3A%22{quote(GITLAB_BASE_URL, safe='')}%2FDataDog%2Fdatadog-agent%2F-%2Fjobs%2F123%22%20%40test.service%3Adatadog-agent",
}
},
{
"external_link": {
"label": "CI Visibility: This job on main",
"url": f"{CI_VISIBILITY_URL}?query=ci_level%3Ajob%20%40ci.job.name%3A%22test-job%22%20%40git.branch%3Amain%20%40ci.pipeline.name%3ADataDog%2Fdatadog-agent",
}
},
{
"external_link": {
"label": "Test Visibility: This job test runs on main",
"url": f"{TEST_VISIBILITY_URL}?query=test_level%3Atest%20%40ci.job.name%3A%22test-job%22%20%40git.branch%3Amain%20%40test.service%3Adatadog-agent",
}
},
]
}
self.assertEqual(
Expand All @@ -44,12 +59,24 @@ def test_create_gitlab_annotations_report_name_with_spaces(self):
"url": f"{CI_VISIBILITY_URL}?query=ci_level%3Ajob%20%40ci.job.id%3A123%20%40ci.pipeline.name%3ADataDog%2Fdatadog-agent",
}
},
{
"external_link": {
"label": "Test Visibility: This job test runs",
"url": f"{TEST_VISIBILITY_URL}?query=test_level%3Atest%20%40ci.job.url%3A%22https%3A%2F%2Fgitlab.ddbuild.io%2FDataDog%2Fdatadog-agent%2F-%2Fjobs%2F123%22%20%40test.service%3Adatadog-agent",
}
},
{
"external_link": {
"label": "CI Visibility: This job on main",
"url": f"{CI_VISIBILITY_URL}?query=ci_level%3Ajob%20%40ci.job.name%3A%22test%20job%22%20%40git.branch%3Amain%20%40ci.pipeline.name%3ADataDog%2Fdatadog-agent",
}
},
{
"external_link": {
"label": "Test Visibility: This job test runs on main",
"url": f"{TEST_VISIBILITY_URL}?query=test_level%3Atest%20%40ci.job.name%3A%22test%20job%22%20%40git.branch%3Amain%20%40test.service%3Adatadog-agent",
}
},
]
}
self.assertEqual(
Expand All @@ -69,12 +96,24 @@ def test_create_gitlab_annotations_report_name_with_weird_chars(self):
"url": f"{CI_VISIBILITY_URL}?query=ci_level%3Ajob%20%40ci.job.id%3A123%20%40ci.pipeline.name%3ADataDog%2Fdatadog-agent",
}
},
{
"external_link": {
"label": "Test Visibility: This job test runs",
"url": f"{TEST_VISIBILITY_URL}?query=test_level%3Atest%20%40ci.job.url%3A%22https%3A%2F%2Fgitlab.ddbuild.io%2FDataDog%2Fdatadog-agent%2F-%2Fjobs%2F123%22%20%40test.service%3Adatadog-agent",
}
},
{
"external_link": {
"label": "CI Visibility: This job on main",
"url": f"{CI_VISIBILITY_URL}?query=ci_level%3Ajob%20%40ci.job.name%3A%22test%20job%20%5BOne%7CTwo%7CThree%5D%20--skip%20Four%22%20%40git.branch%3Amain%20%40ci.pipeline.name%3ADataDog%2Fdatadog-agent",
}
},
{
"external_link": {
"label": "Test Visibility: This job test runs on main",
"url": f"{TEST_VISIBILITY_URL}?query=test_level%3Atest%20%40ci.job.name%3A%22test%20job%20%5BOne%7CTwo%7CThree%5D%20--skip%20Four%22%20%40git.branch%3Amain%20%40test.service%3Adatadog-agent",
}
},
]
}
self.assertEqual(
Expand Down
Loading