From d7d8b6f817194a91808cb0f90edcdf14970c450c Mon Sep 17 00:00:00 2001 From: Tjaz Erzen Date: Wed, 1 Oct 2025 14:42:23 +0200 Subject: [PATCH 1/5] Fix error in conformance test script --- run_devrev_snapin_conformance_tests.sh | 29 ++++++++++++++++---------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/run_devrev_snapin_conformance_tests.sh b/run_devrev_snapin_conformance_tests.sh index bc66301..f04d506 100755 --- a/run_devrev_snapin_conformance_tests.sh +++ b/run_devrev_snapin_conformance_tests.sh @@ -164,23 +164,30 @@ trap cleanup EXIT SIGINT SIGTERM check_and_kill_node_server 8000 check_and_kill_node_server 8002 -# Start the mock DevRev server if it's not already running -if ! lsof -i :8003 -t >/dev/null 2>&1; then - start_mock_devrev_server -else - printf "Mock DevRev server is already running on port 8003\n" - MOCK_SERVER_PID=$(lsof -i :8003 -t) +# Ensure nothing is running on port 8003, then start the mock DevRev server +existing_pids=$(lsof -i :8003 -t 2>/dev/null) +if [ ! -z "$existing_pids" ]; then + printf "Killing existing process(es) on port 8003: %s\n" "$existing_pids" + for pid in $existing_pids; do + kill $pid 2>/dev/null + done + sleep 1 fi +start_mock_devrev_server # Set HTTPS_PROXY environment variable to point to proxy server export HTTPS_PROXY="http://localhost:8004" -if ! lsof -i :8004 -t >/dev/null 2>&1; then - start_proxy_server -else - printf "Proxy server is already running on port 8004\n" - PROXY_SERVER_PID=$(lsof -i :8004 -t) +# Ensure nothing is running on port 8004, then start the proxy server +existing_pids_8004=$(lsof -i :8004 -t 2>/dev/null) +if [ ! -z "$existing_pids_8004" ]; then + printf "Killing existing process(es) on port 8004: %s\n" "$existing_pids_8004" + for pid in $existing_pids_8004; do + kill $pid 2>/dev/null + done + sleep 1 fi +start_proxy_server # Check if chef-cli binary exists at CHEF_CLI_PATH if [ -z "$CHEF_CLI_PATH" ] || [ ! -f "$CHEF_CLI_PATH" ] || [ ! -x "$CHEF_CLI_PATH" ]; then From 1b7cb013b26b74613ae3ed12b01295dba132d644 Mon Sep 17 00:00:00 2001 From: Tjaz Erzen Date: Tue, 7 Oct 2025 17:49:14 +0200 Subject: [PATCH 2/5] Streamline acceptance test templating --- devrev-snapin-template.plain | 1 + run_devrev_snapin_conformance_tests.sh | 6 +++--- ...ttachment_extraction_acceptance_test.plain | 20 +++++++++---------- .../chef_cli_normalization_validation.plain | 20 +++++++++---------- .../data_extraction_acceptance_test.plain | 12 +++++------ ...rate_limiting_during_data_extraction.plain | 12 +++++------ ...rate_limiting_during_emitting_events.plain | 10 +++++----- 7 files changed, 41 insertions(+), 40 deletions(-) diff --git a/devrev-snapin-template.plain b/devrev-snapin-template.plain index b862479..4f9d041 100644 --- a/devrev-snapin-template.plain +++ b/devrev-snapin-template.plain @@ -60,6 +60,7 @@ - The Conformance Tests should spin a server at http://localhost:8002 for testing callbacks (The Callback Server). - The Conformance Tests *MUST NOT* send the event directly to The Callback Server. + - The Implementation Code *MUST NOT* send the event directly to The Callback Server. - The DevRev Server runs at http://localhost:8003, so it must not be mocked. diff --git a/run_devrev_snapin_conformance_tests.sh b/run_devrev_snapin_conformance_tests.sh index f04d506..5491dfc 100755 --- a/run_devrev_snapin_conformance_tests.sh +++ b/run_devrev_snapin_conformance_tests.sh @@ -6,9 +6,9 @@ NPM_INSTALL_OUTPUT_FILTER="up to date in|added [0-9]* packages, removed [0-9]* p ANSI_ESCAPE_PATTERN="s/\x1b\[[0-9;]*[mK]//g" # Maximum number of characters to display from log files -SNAP_IN_LOG_MAX_CHARS=30000 -DEVREV_SERVER_LOG_MAX_CHARS=15000 -PROXY_SERVER_LOG_MAX_CHARS=10000 +SNAP_IN_LOG_MAX_CHARS=80000 +DEVREV_SERVER_LOG_MAX_CHARS=40000 +PROXY_SERVER_LOG_MAX_CHARS=20000 # Function to print a log file, truncating it if it's too large print_log_file() { diff --git a/templates/attachment_extraction_acceptance_test.plain b/templates/attachment_extraction_acceptance_test.plain index e6e91fe..156a185 100644 --- a/templates/attachment_extraction_acceptance_test.plain +++ b/templates/attachment_extraction_acceptance_test.plain @@ -1,10 +1,10 @@ - - Test The Extraction Function with the following flow (The Attachment Test Flow): - - Step 1: Invoke The Extraction Function using the resource [{{ data_extraction_resource_name }}](test_data/{{ data_extraction_resource_name }}). - - Expect The Callback Server to receive from DevRev a **single** event with "event_type" that equals "EXTRACTION_DATA_DONE". - - Step 2: Invoke The Extraction Function using the resource [{{ attachment_extraction_resource_name }}](test_data/{{ attachment_extraction_resource_name }}). - - Expect The Callback Server to receive from DevRev a **single** event (`:= event_from_callback_server`) with "event_type" that equals "EXTRACTION_ATTACHMENTS_DONE". - - Expect `event_from_callback_server["event_data"]["artifacts"]` to be an array. Expect this array to not be empty. Expect this array to have length 1. - - Let `artifact_object = event_from_callback_server["event_data"]["artifacts"][0]`. - - Expect `artifact_object["item_type"] = "ssor_attachment"` and `artifact_object["item_count"] = {{ expected_attachment_count }}`. - - Make a GET request to "http://localhost:8003/is_uploaded/{artifact_object['id']}". Expect response code to be equal to 200. - - Note: The Attachment Test Flow should be executed within a single test. \ No newline at end of file +Test The Extraction Function with the following flow (The Attachment Test Flow): + - Step 1: Invoke The Extraction Function using the resource [{{ data_extraction_resource_name }}](test_data/{{ data_extraction_resource_name }}). + - Expect The Callback Server to receive from DevRev a **single** event with "event_type" that equals "EXTRACTION_DATA_DONE". + - Step 2: Invoke The Extraction Function using the resource [{{ attachment_extraction_resource_name }}](test_data/{{ attachment_extraction_resource_name }}). + - Expect The Callback Server to receive from DevRev a **single** event (`:= event_from_callback_server`) with "event_type" that equals "EXTRACTION_ATTACHMENTS_DONE". + - Expect `event_from_callback_server["event_data"]["artifacts"]` to be an array. Expect this array to not be empty. Expect this array to have length 1. + - Let `artifact_object = event_from_callback_server["event_data"]["artifacts"][0]`. + - Expect `artifact_object["item_type"] = "ssor_attachment"` and `artifact_object["item_count"] = {{ expected_attachment_count }}`. + - Make a GET request to "http://localhost:8003/is_uploaded/{artifact_object['id']}". Expect response code to be equal to 200. + - Note: The Attachment Test Flow should be executed within a single test. \ No newline at end of file diff --git a/templates/chef_cli_normalization_validation.plain b/templates/chef_cli_normalization_validation.plain index 19a8cb8..101f428 100644 --- a/templates/chef_cli_normalization_validation.plain +++ b/templates/chef_cli_normalization_validation.plain @@ -1,10 +1,10 @@ - - Then, validate The Normalization Function with the chef-cli tool (The Chef CLI) using the command "chef-cli validate-data -m -r {{ record_type }}": - - To invoke the function that returns The External Domain Metadata JSON object, send the event [external_domain_metadata_event_payload.json](test_data/external_domain_metadata_event_payload.json) to The Snap-In server. - - The External Domain Metadata JSON object should then be stored in temporary file and provided as path to the validation of The Normalization Function. - - The Chef CLI reads the extracted file (The Extracted File) from stdin. - - The Extracted File is inside folder provided in the environment variable "EXTRACTED_FILES_FOLDER_PATH". To retrieve the relevant file name, command `ls EXTRACTED_FILES_FOLDER_PATH | grep extractor_{{ record_type }} | sort -r | head -n 1` should be executed. The test should always fail if The Extracted File is not present. - - Please note that the expected behaviour is that only after the invocation of The Extraction Function, the folder provided in the environment variable "EXTRACTED_FILES_FOLDER_PATH" should exist. The test should always fail if before the execution of The Chef CLI, the folder "EXTRACTED_FILES_FOLDER_PATH" does not exist or if The Extracted File is not found. - - The path to The Chef CLI executable is provided in the environment variable `CHEF_CLI_PATH`. - - For the validation of The Normalization Function to be successful The Chef CLI must return an empty output. The implementation of The Extraction Function directly infuences the output of The Chef CLI. - - The stdout and stderr of The Chef CLI *must* be printed to the console. - - The test should always fail if The Chef CLI is not available. \ No newline at end of file +validate The Normalization Function with the chef-cli tool (The Chef CLI) using the command "chef-cli validate-data -m -r {{ record_type }}": + - To invoke the function that returns The External Domain Metadata JSON object, send the event [external_domain_metadata_event_payload.json](test_data/external_domain_metadata_event_payload.json) to The Snap-In server. + - The External Domain Metadata JSON object should then be stored in temporary file and provided as path to the validation of The Normalization Function. + - The Chef CLI reads the extracted file (The Extracted File) from stdin. + - The Extracted File is inside folder provided in the environment variable "EXTRACTED_FILES_FOLDER_PATH". To retrieve the relevant file name, command `ls EXTRACTED_FILES_FOLDER_PATH | grep extractor_{{ record_type }} | sort -r | head -n 1` should be executed. The test should always fail if The Extracted File is not present. + - Please note that the expected behaviour is that only after the invocation of The Extraction Function, the folder provided in the environment variable "EXTRACTED_FILES_FOLDER_PATH" should exist. The test should always fail if before the execution of The Chef CLI, the folder "EXTRACTED_FILES_FOLDER_PATH" does not exist or if The Extracted File is not found. + - The path to The Chef CLI executable is provided in the environment variable `CHEF_CLI_PATH`. + - For the validation of The Normalization Function to be successful The Chef CLI must return an empty output. The implementation of The Extraction Function directly infuences the output of The Chef CLI. + - The stdout and stderr of The Chef CLI *must* be printed to the console. + - The test should always fail if The Chef CLI is not available. \ No newline at end of file diff --git a/templates/data_extraction_acceptance_test.plain b/templates/data_extraction_acceptance_test.plain index 012b89d..c517aca 100644 --- a/templates/data_extraction_acceptance_test.plain +++ b/templates/data_extraction_acceptance_test.plain @@ -1,6 +1,6 @@ - - Test The Extraction Function using the resource [{{ resource_name }}](test_data/{{ resource_name }}). - - Expect The Callback Server to receive from DevRev a **single** event with "event_type" that equals "EXTRACTION_DATA_DONE". - - Let `artifact_array = ["event_data"]["artifacts"]` (array of objects). - - Expect `len(artifact_array) > 0`. - - Out of `artifact_array`, expect one of the elements to have "item_type" equal to "{{ artifact_name }}" (`:= {{ artifact_name }}_artifact`) and `{{ artifact_name }}_artifact["item_count"]={{ expected_item_count }}`. - - Note: If `{{ artifact_name }}_artifact["item_count"] < {{ expected_item_count }}`, this indicates that not all the {{ artifact_name }} data was extracted. \ No newline at end of file +Test The Extraction Function using the resource [{{ resource_name }}](test_data/{{ resource_name }}). + - Expect The Callback Server to receive from DevRev a **single** event with "event_type" that equals "EXTRACTION_DATA_DONE". + - Let `artifact_array = ["event_data"]["artifacts"]` (array of objects). + - Expect `len(artifact_array) > 0`. + - Out of `artifact_array`, expect one of the elements to have "item_type" equal to "{{ artifact_name }}" (`:= {{ artifact_name }}_artifact`) and `{{ artifact_name }}_artifact["item_count"]={{ expected_item_count }}`. + - Note: If `{{ artifact_name }}_artifact["item_count"] < {{ expected_item_count }}`, this indicates that not all the {{ artifact_name }} data was extracted. \ No newline at end of file diff --git a/templates/test_rate_limiting_during_data_extraction.plain b/templates/test_rate_limiting_during_data_extraction.plain index ee183fd..64ed389 100644 --- a/templates/test_rate_limiting_during_data_extraction.plain +++ b/templates/test_rate_limiting_during_data_extraction.plain @@ -1,7 +1,7 @@ - - Test the rate limiting of The Function "{{ function_name }}" with the following flow: - - Make a POST request to "http://localhost:8004/start_rate_limiting" with body `{ "test_name": }`. - - Invoke The Function "{{ function_name }}" with valid credentials and all required parameters. - - Expect: `status_code = 429`. - - Expect: `api_delay > 0` and `api_delay <= 3`. If api_delay is bigger than 3, assume the problem is that we did not calculate the api_delay correctly in The Implementation Code. - - Make a POST request to "http://localhost:8004/end_rate_limiting". +Test the rate limiting of The Function "{{ function_name }}" with the following flow: + - Make a POST request to "http://localhost:8004/start_rate_limiting" with body `{ "test_name": }`. + - Invoke The Function "{{ function_name }}" with valid credentials and all required parameters. + - Expect: `status_code = 429`. + - Expect: `api_delay > 0` and `api_delay <= 3`. If api_delay is bigger than 3, assume the problem is that we did not calculate the api_delay correctly in The Implementation Code. + - Make a POST request to "http://localhost:8004/end_rate_limiting". \ No newline at end of file diff --git a/templates/test_rate_limiting_during_emitting_events.plain b/templates/test_rate_limiting_during_emitting_events.plain index c01ecb6..7744647 100644 --- a/templates/test_rate_limiting_during_emitting_events.plain +++ b/templates/test_rate_limiting_during_emitting_events.plain @@ -1,5 +1,5 @@ - - When the input event is "{{ input_event_name }}", the Extraction Function must handle rate limiting using the following test flow: - - Step 1: Make a POST request to "http://localhost:8004/start_rate_limiting" with body `{ "test_name": }`. - - Step 2: Invoke The Extraction Function using the resource [{{ resource_name }}](test_data/{{ resource_name }}). - - Expect The Callback Server to receive *a single* event with "event_type" "{{ expected_output_event_type }}". - - Step 3: Make a POST request to "http://localhost:8004/end_rate_limiting". \ No newline at end of file +When the input event is "{{ input_event_name }}", the Extraction Function must handle rate limiting using the following test flow: + - Step 1: Make a POST request to "http://localhost:8004/start_rate_limiting" with body `{ "test_name": }`. + - Step 2: Invoke The Extraction Function using the resource [{{ resource_name }}](test_data/{{ resource_name }}). + - Expect The Callback Server to receive *a single* event with "event_type" "{{ expected_output_event_type }}". + - Step 3: Make a POST request to "http://localhost:8004/end_rate_limiting". \ No newline at end of file From 928e31ddb163c81f762a7435f2a55ad517ae0a48 Mon Sep 17 00:00:00 2001 From: Tjaz Erzen Date: Tue, 7 Oct 2025 21:01:12 +0200 Subject: [PATCH 3/5] Update base folder error handling --- base_folder/test/runner.ts | 60 +++++++++++++++++++++++++++++++++++--- 1 file changed, 56 insertions(+), 4 deletions(-) diff --git a/base_folder/test/runner.ts b/base_folder/test/runner.ts index 6a392b9..5d78707 100644 --- a/base_folder/test/runner.ts +++ b/base_folder/test/runner.ts @@ -75,6 +75,18 @@ async function handleEvent(events: any[], isAsync: boolean, resp: Response) { resp.status(400).send(errMsg); return; } + // crash the process if an empty array is provided + if (Array.isArray(events) && events.length === 0) { + let errMsg = 'Invalid request format: body is an empty array'; + error = { + err_type: RuntimeErrorType.InvalidRequest, + err_msg: errMsg, + } as RuntimeError; + console.error(error.err_msg); + // Return validation error status for empty events input + resp.status(400).send(errMsg); + return; + } // if the request is synchronous, there should be a single event if (!isAsync) { if (events.length > 1) { @@ -88,6 +100,29 @@ async function handleEvent(events: any[], isAsync: boolean, resp: Response) { return; } } else { + // Preflight validation for async requests: ensure each event is minimally valid + for (let event of events) { + if (!event || !event.execution_metadata) { + let errMsg = 'Invalid request format: missing execution_metadata'; + console.error(errMsg); + resp.status(400).send(errMsg); + return; + } + const functionName: FunctionFactoryType = event.execution_metadata.function_name as FunctionFactoryType; + if (functionName === undefined) { + let errMsg = 'Function name not provided in event'; + console.error(errMsg); + resp.status(400).send(errMsg); + return; + } + const f = functionFactory[functionName]; + if (f == undefined) { + let errMsg = `Function ${event.execution_metadata.function_name} not found in factory`; + console.error(errMsg); + resp.status(400).send(errMsg); + return; + } + } // return a success response back to the server resp.status(200).send(); } @@ -101,8 +136,13 @@ async function handleEvent(events: any[], isAsync: boolean, resp: Response) { err_msg: errMsg, } as RuntimeError; console.error(error.err_msg); - resp.status(400).send(errMsg); - return; + if (!isAsync) { + resp.status(400).send(errMsg); + return; + } else { + // For async requests, response has already been sent; skip further processing for this event + continue; + } } const functionName: FunctionFactoryType = event.execution_metadata.function_name as FunctionFactoryType; if (functionName === undefined) { @@ -111,7 +151,13 @@ async function handleEvent(events: any[], isAsync: boolean, resp: Response) { err_msg: 'Function name not provided in event', } as RuntimeError; console.error(error.err_msg); - receivedError = true; + if (!isAsync) { + resp.status(400).send(error.err_msg); + return; + } else { + receivedError = true; + continue; + } } else { const f = functionFactory[functionName]; try { @@ -121,7 +167,13 @@ async function handleEvent(events: any[], isAsync: boolean, resp: Response) { err_msg: `Function ${event.execution_metadata.function_name} not found in factory`, } as RuntimeError; console.error(error.err_msg); - receivedError = true; + if (!isAsync) { + resp.status(400).send(error.err_msg); + return; + } else { + receivedError = true; + continue; + } } else { result = await run(f, [event]); } From f7eb9e55901be2ffad7a79207c2597cb45f38493 Mon Sep 17 00:00:00 2001 From: Tjaz Erzen Date: Tue, 14 Oct 2025 12:07:24 +0200 Subject: [PATCH 4/5] Revert base folder runner --- base_folder/test/runner.ts | 60 +++----------------------------------- 1 file changed, 4 insertions(+), 56 deletions(-) diff --git a/base_folder/test/runner.ts b/base_folder/test/runner.ts index 5d78707..6a392b9 100644 --- a/base_folder/test/runner.ts +++ b/base_folder/test/runner.ts @@ -75,18 +75,6 @@ async function handleEvent(events: any[], isAsync: boolean, resp: Response) { resp.status(400).send(errMsg); return; } - // crash the process if an empty array is provided - if (Array.isArray(events) && events.length === 0) { - let errMsg = 'Invalid request format: body is an empty array'; - error = { - err_type: RuntimeErrorType.InvalidRequest, - err_msg: errMsg, - } as RuntimeError; - console.error(error.err_msg); - // Return validation error status for empty events input - resp.status(400).send(errMsg); - return; - } // if the request is synchronous, there should be a single event if (!isAsync) { if (events.length > 1) { @@ -100,29 +88,6 @@ async function handleEvent(events: any[], isAsync: boolean, resp: Response) { return; } } else { - // Preflight validation for async requests: ensure each event is minimally valid - for (let event of events) { - if (!event || !event.execution_metadata) { - let errMsg = 'Invalid request format: missing execution_metadata'; - console.error(errMsg); - resp.status(400).send(errMsg); - return; - } - const functionName: FunctionFactoryType = event.execution_metadata.function_name as FunctionFactoryType; - if (functionName === undefined) { - let errMsg = 'Function name not provided in event'; - console.error(errMsg); - resp.status(400).send(errMsg); - return; - } - const f = functionFactory[functionName]; - if (f == undefined) { - let errMsg = `Function ${event.execution_metadata.function_name} not found in factory`; - console.error(errMsg); - resp.status(400).send(errMsg); - return; - } - } // return a success response back to the server resp.status(200).send(); } @@ -136,13 +101,8 @@ async function handleEvent(events: any[], isAsync: boolean, resp: Response) { err_msg: errMsg, } as RuntimeError; console.error(error.err_msg); - if (!isAsync) { - resp.status(400).send(errMsg); - return; - } else { - // For async requests, response has already been sent; skip further processing for this event - continue; - } + resp.status(400).send(errMsg); + return; } const functionName: FunctionFactoryType = event.execution_metadata.function_name as FunctionFactoryType; if (functionName === undefined) { @@ -151,13 +111,7 @@ async function handleEvent(events: any[], isAsync: boolean, resp: Response) { err_msg: 'Function name not provided in event', } as RuntimeError; console.error(error.err_msg); - if (!isAsync) { - resp.status(400).send(error.err_msg); - return; - } else { - receivedError = true; - continue; - } + receivedError = true; } else { const f = functionFactory[functionName]; try { @@ -167,13 +121,7 @@ async function handleEvent(events: any[], isAsync: boolean, resp: Response) { err_msg: `Function ${event.execution_metadata.function_name} not found in factory`, } as RuntimeError; console.error(error.err_msg); - if (!isAsync) { - resp.status(400).send(error.err_msg); - return; - } else { - receivedError = true; - continue; - } + receivedError = true; } else { result = await run(f, [event]); } From 9f20f0d3dee72d5383ecf0a4ae903e2a91da8c9d Mon Sep 17 00:00:00 2001 From: Tjaz Erzen Date: Tue, 14 Oct 2025 15:03:12 +0200 Subject: [PATCH 5/5] Update devrev snapin template --- devrev-snapin-template.plain | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/devrev-snapin-template.plain b/devrev-snapin-template.plain index 6cc2bea..7da474a 100644 --- a/devrev-snapin-template.plain +++ b/devrev-snapin-template.plain @@ -74,6 +74,14 @@ - The Conformance Tests should send a single event to the Snap-In Server in the request body without wrapping it in an array. - It is safe to assume that event payload will never be undefined, null, or empty. +- Do not invoke The Function with: + - "function_name" that doesn't exist. + - Empty event payload. + - "None" event payload. + - Missing "execution_metadata" field. + +- More generally, for any unexpected events sent to The Snap-In Server, do not rely on the returned status codes being correct or meaningful. + - If generating The Acceptance Test that references a JSON file (The Attached JSON Of The Acceptance Test) in the resource, you should: - Store The Attached JSON Of The Acceptance Test in a separate JSON file. - Make sure to replace placeholders of the credentials (in `event["payload"]["connection_data"]["key"]` and `event["payload"]["connection_data"]["org_id"]`) in The Attached JSON Of The Acceptance Test with the actual values (credentials from The API).