Skip to content

[FLINK-34047][ci] Make CI scripts agnostic to CI environment #24061

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
XComp merged 2 commits into from
Jan 12, 2024
Merged

[FLINK-34047][ci] Make CI scripts agnostic to CI environment #24061

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
81b59d1
[FLINK-34047][ci] Makes prepare_debug_files not rely on "magic" envir…
XComp Jan 10, 2024
91a76e9
[FLINK-34047][ci] Makes uploading_watchdog.sh support Azure Pipelines…
XComp Jan 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 11 additions & 3 deletions tools/azure-pipelines/debug_files_utils.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,17 @@
################################################################################

function prepare_debug_files {
MODULE=$@
export DEBUG_FILES_OUTPUT_DIR="$AGENT_TEMPDIRECTORY/debug_files"
export DEBUG_FILES_NAME="$(echo $MODULE | tr -c '[:alnum:]\n\r' '_')-$(date +%s)"
if [ "$#" != "2" ]; then
echo "[ERROR] Invalid number of parameters passed. Expected parameters for $0: <parent-directory> <module-label>"
exit 1
fi

local parent_directory module
parent_directory="$1"
module="$2"

export DEBUG_FILES_OUTPUT_DIR="${parent_directory}/debug_files"
export DEBUG_FILES_NAME="$(echo "${module}" | tr -c '[:alnum:]\n\r' '_')-$(date +%s)"
echo "##vso[task.setvariable variable=DEBUG_FILES_OUTPUT_DIR]$DEBUG_FILES_OUTPUT_DIR"
echo "##vso[task.setvariable variable=DEBUG_FILES_NAME]$DEBUG_FILES_NAME"
mkdir -p $DEBUG_FILES_OUTPUT_DIR || { echo "FAILURE: cannot create debug files directory '${DEBUG_FILES_OUTPUT_DIR}'." ; exit 1; }
Expand Down
33 changes: 31 additions & 2 deletions tools/azure-pipelines/uploading_watchdog.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,16 +27,45 @@ if [ -z "$HERE" ] ; then
exit 1
fi

if [ -n "${TF_BUILD+x}" ]; then
echo "[INFO] Azure Pipelines environment detected: $0 will rely on Azure-specific environment variables."
job_name="$AGENT_JOBNAME"
temporary_folder="$AGENT_TEMPDIRECTORY"
job_timeout="$SYSTEM_JOBTIMEOUT"
pipeline_start_time="${SYSTEM_PIPELINESTARTTIME}"
elif [ -n "${GITHUB_ACTIONS+x}" ]; then
echo "[INFO] GitHub Actions environment detected: $0 will rely on GHA-specific environment variables."
job_name="${GITHUB_JOB}"
temporary_folder="${RUNNER_TEMP}"

if [ -n "${GHA_JOB_TIMEOUT+x}" ]; then
job_timeout="${GHA_JOB_TIMEOUT}"
else
echo "[ERROR] GHA_JOB_TIMEOUT is not set: GHA_JOB_TIMEOUT needs to be set by the workflow because there is no pre-defined environment variable available for this parameter for now."
exit 1
fi

if [ -n "${GHA_PIPELINE_START_TIME+x}" ]; then
pipeline_start_time="${GHA_PIPELINE_START_TIME}"
else
echo "[ERROR] GHA_PIPELINE_START_TIME is not set: GHA_PIPELINE_START_TIME needs to be set by the workflow because there is no pre-defined environment variable available for this parameter for now."
exit 1
fi
else
echo "[ERROR] No CI environment detected that could be used for injecting the parameters which are needed by $0."
exit 1
fi

source "${HERE}/../ci/controller_utils.sh"

source ./tools/azure-pipelines/debug_files_utils.sh
prepare_debug_files "$AGENT_JOBNAME"
prepare_debug_files "${temporary_folder}" "${job_name}"
export FLINK_LOG_DIR="$DEBUG_FILES_OUTPUT_DIR/flink-logs"
mkdir $FLINK_LOG_DIR || { echo "FAILURE: cannot create log directory '${FLINK_LOG_DIR}'." ; exit 1; }
sudo apt-get install -y moreutils

REAL_START_SECONDS=$(date +"%s")
REAL_END_SECONDS=$(date -d "$SYSTEM_PIPELINESTARTTIME + $SYSTEM_JOBTIMEOUT minutes" +"%s")
REAL_END_SECONDS=$(date -d "${pipeline_start_time} + ${job_timeout} minutes" +"%s")
REAL_TIMEOUT_SECONDS=$(($REAL_END_SECONDS - $REAL_START_SECONDS))
KILL_SECONDS_BEFORE_TIMEOUT=$((2 * 60))

Expand Down