[Qualys GAV] - Added standard API rate limit headers in requests

packages/qualys_gav/_dev/deploy/docker/files/config.yml

@@ -6,6 +6,10 @@ rules:
         headers:
           Content-Type:
             - 'application/json'
+          x-ratelimit-limit: ["100"]
+          x-ratelimit-remaining: ["99"]
+          x-ratelimit-window-sec: ["3600"]
+          x-ratelimit-towait-sec: ["0"]
         body: "xxxx"
   - path: /rest/2.0/search/am/asset
     methods: ['POST']
@@ -17,6 +21,11 @@ rules:
         - 'Bearer xxxx'
     responses:
       - status_code: 200
+        headers:
+          x-ratelimit-limit: ["100"]
+          x-ratelimit-remaining: ["98"]
+          x-ratelimit-window-sec: ["3600"]
+          x-ratelimit-towait-sec: ["0"]
         body: |
           {{ minify_json `
           {
@@ -892,6 +901,11 @@ rules:
         - 'Bearer xxxx'
     responses:
       - status_code: 200
+        headers:
+          x-ratelimit-limit: ["100"]
+          x-ratelimit-remaining: ["98"]
+          x-ratelimit-window-sec: ["3600"]
+          x-ratelimit-towait-sec: ["0"]
         body: |
           {{ minify_json `
           {
@@ -1697,6 +1711,11 @@ rules:
         - 'Bearer xxxx'
     responses:
       - status_code: 200
+        headers:
+          x-ratelimit-limit: ["100"]
+          x-ratelimit-remaining: ["98"]
+          x-ratelimit-window-sec: ["3600"]
+          x-ratelimit-towait-sec: ["0"]
         body: |
           {{ minify_json `
           {

packages/qualys_gav/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.5.0"
+  changes:
+    - description: Implement X-RateLimit header handling for API rate limiting.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/16019
 - version: "0.4.1"
   changes:
     - description: Refactor date processing for asset inventory list to use foreach processor.

packages/qualys_gav/data_stream/asset/agent/stream/cel.yml.hbs

@@ -38,7 +38,7 @@ program: |
         base_url + "/auth",
         "application/x-www-form-urlencoded",
         {"username":[state.username],"password":[state.password]}.format_query()
-      ).do_request().as(resp, resp.StatusCode == 201 ?
+      ).do_request().as(resp, (resp.StatusCode == 201 ?
         {
           "access_token": string(resp.Body),
           // Include 30s grace period to manage session expiry.
@@ -60,7 +60,43 @@ program: |
           },
           "want_more": false,
         }
-      )
+      ).with(
+        resp.Header.transformMapEntry(k, v,
+          // Canonicalise header keys to match rate_limit conventions.
+          // -Limit, -Remaining and -Reset are magic suffixes in rate_limit.
+          {
+            k.has_suffix("-Limit") ?
+              (k.trim_suffix("-Limit").to_lower() + "-Limit")
+            : k.has_suffix("-Remaining") ?
+              (k.trim_suffix("-Remaining").to_lower() + "-Remaining")
+            :
+              k.to_lower(): v,
+          }
+        ).as(headers,
+          // Calculate rate limits.
+          rate_limit(
+            headers.with(
+              {
+                "x-ratelimit-Reset": [string(headers[?"x-ratelimit-towait-sec"][0].orValue("3600"))],
+              }
+            ),
+            "x-ratelimit",
+            false,
+            true,
+            duration(string(headers[?"x-ratelimit-window-sec"][0].orValue("3600")) + "s"),
+            0
+          )
+        ).as(rate_headers, rate_headers.with({
+          // Work around inf detection in input.
+          // If the headers are missing or rate_limit failed, rate and
+          // next may be missing. So use optional types.
+          ?"rate": rate_headers.?rate == optional.of(double("Infinity")) ? optional.of("inf") : optional.none(),
+          ?"next": rate_headers.?next == optional.of(double("Infinity")) ? optional.of("inf") : optional.none(),
+        })).as(limit, {
+          "header": resp.Header,
+          "rate_limit": limit,
+        })
+      ))
     ).as(token,
       has(token.events) ? token : // Exit early due to failure.
         request(
@@ -73,7 +109,7 @@ program: |
           "Header":{
             "Authorization": ["Bearer " + token.access_token],
           }
-        }).do_request().as(resp, resp.StatusCode == 200 ?
+        }).do_request().as(resp, (resp.StatusCode == 200 ?
           resp.Body.decode_json().as(body, {
             "events": body.assetListData.asset.map(e,{
               "message": e.encode_json(),
@@ -109,7 +145,43 @@ program: |
               },
               "want_more": false,
             }
-        )
+        ).with(
+          resp.Header.transformMapEntry(k, v,
+            // Canonicalise header keys to match rate_limit conventions.
+            // -Limit, -Remaining and -Reset are magic suffixes in rate_limit.
+            {
+              k.has_suffix("-Limit") ?
+                (k.trim_suffix("-Limit").to_lower() + "-Limit")
+              : k.has_suffix("-Remaining") ?
+                (k.trim_suffix("-Remaining").to_lower() + "-Remaining")
+              :
+                k.to_lower(): v,
+            }
+          ).as(headers,
+            // Calculate rate limits.
+            rate_limit(
+              headers.with(
+                {
+                  "x-ratelimit-Reset": [string(headers[?"x-ratelimit-towait-sec"][0].orValue("3600"))],
+                }
+              ),
+              "x-ratelimit",
+              false,
+              true,
+              duration(string(headers[?"x-ratelimit-window-sec"][0].orValue("3600")) + "s"),
+              0
+            )
+          ).as(rate_headers, rate_headers.with({
+            // Work around inf detection in input.
+            // If the headers are missing or rate_limit failed, rate and
+            // next may be missing. So use optional types.
+            ?"rate": rate_headers.?rate == optional.of(double("Infinity")) ? optional.of("inf") : optional.none(),
+            ?"next": rate_headers.?next == optional.of(double("Infinity")) ? optional.of("inf") : optional.none(),
+          })).as(limit, {
+            "header": resp.Header,
+            "rate_limit": limit,
+          })
+        ))
     )
   ))
 tags:

packages/qualys_gav/manifest.yml

@@ -1,14 +1,14 @@
 format_version: 3.3.2
 name: qualys_gav
 title: Qualys Global AssetView
-version: 0.4.1
+version: 0.5.0
 description: Collect logs from Qualys Global AssetView with Elastic Agent.
 type: integration
 categories:
   - security
 conditions:
   kibana:
-    version: ^8.18.5 || ^8.19.2  || ^9.0.5  || ^9.1.2
+    version: ^8.19.2 || ^9.1.2
   elastic:
     subscription: basic
 screenshots: