From 162124b4a3872186b44019a0680d69adb9971256 Mon Sep 17 00:00:00 2001 From: Shourie Ganguly Date: Wed, 19 Nov 2025 15:28:44 +0530 Subject: [PATCH 1/3] added rate limit support --- .../_dev/deploy/docker/files/config.yml | 19 +++++ packages/qualys_gav/changelog.yml | 5 ++ .../asset/agent/stream/cel.yml.hbs | 80 ++++++++++++++++++- packages/qualys_gav/manifest.yml | 2 +- 4 files changed, 101 insertions(+), 5 deletions(-) diff --git a/packages/qualys_gav/_dev/deploy/docker/files/config.yml b/packages/qualys_gav/_dev/deploy/docker/files/config.yml index 9689333381f..2fda162e2a4 100644 --- a/packages/qualys_gav/_dev/deploy/docker/files/config.yml +++ b/packages/qualys_gav/_dev/deploy/docker/files/config.yml @@ -6,6 +6,10 @@ rules: headers: Content-Type: - 'application/json' + x-ratelimit-limit: ["100"] + x-ratelimit-remaining: ["99"] + x-ratelimit-window-sec: ["3600"] + x-ratelimit-towait-sec: ["0"] body: "xxxx" - path: /rest/2.0/search/am/asset methods: ['POST'] @@ -17,6 +21,11 @@ rules: - 'Bearer xxxx' responses: - status_code: 200 + headers: + x-ratelimit-limit: ["100"] + x-ratelimit-remaining: ["98"] + x-ratelimit-window-sec: ["3600"] + x-ratelimit-towait-sec: ["0"] body: | {{ minify_json ` { @@ -892,6 +901,11 @@ rules: - 'Bearer xxxx' responses: - status_code: 200 + headers: + x-ratelimit-limit: ["100"] + x-ratelimit-remaining: ["98"] + x-ratelimit-window-sec: ["3600"] + x-ratelimit-towait-sec: ["0"] body: | {{ minify_json ` { @@ -1697,6 +1711,11 @@ rules: - 'Bearer xxxx' responses: - status_code: 200 + headers: + x-ratelimit-limit: ["100"] + x-ratelimit-remaining: ["98"] + x-ratelimit-window-sec: ["3600"] + x-ratelimit-towait-sec: ["0"] body: | {{ minify_json ` { diff --git a/packages/qualys_gav/changelog.yml b/packages/qualys_gav/changelog.yml index f05ec2697c0..6843453009f 100644 --- a/packages/qualys_gav/changelog.yml +++ b/packages/qualys_gav/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.5.0" + changes: + - description: Implement X-RateLimit header handling for API rate limiting. + type: enhancement + link: https://github.com/elastic/integrations/pull/1111 - version: "0.4.1" changes: - description: Refactor date processing for asset inventory list to use foreach processor. diff --git a/packages/qualys_gav/data_stream/asset/agent/stream/cel.yml.hbs b/packages/qualys_gav/data_stream/asset/agent/stream/cel.yml.hbs index 841f3adfd92..48ce51035b6 100644 --- a/packages/qualys_gav/data_stream/asset/agent/stream/cel.yml.hbs +++ b/packages/qualys_gav/data_stream/asset/agent/stream/cel.yml.hbs @@ -38,7 +38,7 @@ program: | base_url + "/auth", "application/x-www-form-urlencoded", {"username":[state.username],"password":[state.password]}.format_query() - ).do_request().as(resp, resp.StatusCode == 201 ? + ).do_request().as(resp, (resp.StatusCode == 201 ? { "access_token": string(resp.Body), // Include 30s grace period to manage session expiry. @@ -60,7 +60,43 @@ program: | }, "want_more": false, } - ) + ).with( + resp.Header.transformMapEntry(k, v, + // Canonicalise header keys to match rate_limit conventions. + // -Limit, -Remaining and -Reset are magic suffixes in rate_limit. + { + k.has_suffix("-Limit") ? + (k.trim_suffix("-Limit").to_lower() + "-Limit") + : k.has_suffix("-Remaining") ? + (k.trim_suffix("-Remaining").to_lower() + "-Remaining") + : + k.to_lower(): v, + } + ).as(headers, + // Calculate rate limits. + rate_limit( + headers.with( + { + "x-ratelimit-Reset": [string(headers[?"x-ratelimit-towait-sec"][0].orValue("3600"))], + } + ), + "x-ratelimit", + false, + true, + duration(string(headers[?"x-ratelimit-window-sec"][0].orValue("3600")) + "s"), + 0 + ) + ).as(rate_headers, rate_headers.with({ + // Work around inf detection in input. + // If the headers are missing or rate_limit failed, rate and + // next may be missing. So use optional types. + ?"rate": rate_headers.?rate == optional.of(double("Infinity")) ? optional.of("inf") : optional.none(), + ?"next": rate_headers.?next == optional.of(double("Infinity")) ? optional.of("inf") : optional.none(), + })).as(limit, { + "header": resp.Header, + "rate_limit": limit, + }) + )) ).as(token, has(token.events) ? token : // Exit early due to failure. request( @@ -73,7 +109,7 @@ program: | "Header":{ "Authorization": ["Bearer " + token.access_token], } - }).do_request().as(resp, resp.StatusCode == 200 ? + }).do_request().as(resp, (resp.StatusCode == 200 ? resp.Body.decode_json().as(body, { "events": body.assetListData.asset.map(e,{ "message": e.encode_json(), @@ -109,7 +145,43 @@ program: | }, "want_more": false, } - ) + ).with( + resp.Header.transformMapEntry(k, v, + // Canonicalise header keys to match rate_limit conventions. + // -Limit, -Remaining and -Reset are magic suffixes in rate_limit. + { + k.has_suffix("-Limit") ? + (k.trim_suffix("-Limit").to_lower() + "-Limit") + : k.has_suffix("-Remaining") ? + (k.trim_suffix("-Remaining").to_lower() + "-Remaining") + : + k.to_lower(): v, + } + ).as(headers, + // Calculate rate limits. + rate_limit( + headers.with( + { + "x-ratelimit-Reset": [string(headers[?"x-ratelimit-towait-sec"][0].orValue("3600"))], + } + ), + "x-ratelimit", + false, + true, + duration(string(headers[?"x-ratelimit-window-sec"][0].orValue("3600")) + "s"), + 0 + ) + ).as(rate_headers, rate_headers.with({ + // Work around inf detection in input. + // If the headers are missing or rate_limit failed, rate and + // next may be missing. So use optional types. + ?"rate": rate_headers.?rate == optional.of(double("Infinity")) ? optional.of("inf") : optional.none(), + ?"next": rate_headers.?next == optional.of(double("Infinity")) ? optional.of("inf") : optional.none(), + })).as(limit, { + "header": resp.Header, + "rate_limit": limit, + }) + )) ) )) tags: diff --git a/packages/qualys_gav/manifest.yml b/packages/qualys_gav/manifest.yml index 798e82a7f10..ed781b07d97 100644 --- a/packages/qualys_gav/manifest.yml +++ b/packages/qualys_gav/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.3.2 name: qualys_gav title: Qualys Global AssetView -version: 0.4.1 +version: 0.5.0 description: Collect logs from Qualys Global AssetView with Elastic Agent. type: integration categories: From d8e29998f3ed8a7cd75f03d9254aef43c0ce5a14 Mon Sep 17 00:00:00 2001 From: Shourie Ganguly Date: Wed, 19 Nov 2025 15:54:57 +0530 Subject: [PATCH 2/3] updated changelog --- packages/qualys_gav/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/qualys_gav/changelog.yml b/packages/qualys_gav/changelog.yml index 6843453009f..2c95386babf 100644 --- a/packages/qualys_gav/changelog.yml +++ b/packages/qualys_gav/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Implement X-RateLimit header handling for API rate limiting. type: enhancement - link: https://github.com/elastic/integrations/pull/1111 + link: https://github.com/elastic/integrations/pull/16019 - version: "0.4.1" changes: - description: Refactor date processing for asset inventory list to use foreach processor. From 4ceeb967094f08dc79334645f492051f9872ff8b Mon Sep 17 00:00:00 2001 From: Shourie Ganguly Date: Thu, 20 Nov 2025 14:38:22 +0530 Subject: [PATCH 3/3] updated stack version to be compatible with CEL methods --- packages/qualys_gav/manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/qualys_gav/manifest.yml b/packages/qualys_gav/manifest.yml index ed781b07d97..5a4a2e8cc1c 100644 --- a/packages/qualys_gav/manifest.yml +++ b/packages/qualys_gav/manifest.yml @@ -8,7 +8,7 @@ categories: - security conditions: kibana: - version: ^8.18.5 || ^8.19.2 || ^9.0.5 || ^9.1.2 + version: ^8.19.2 || ^9.1.2 elastic: subscription: basic screenshots: