Merge branch 'alertlogic:master' into im-merakichanges

collectors/ciscomeraki/README.md

@@ -1,9 +1,9 @@
-# Ciscomeraki collector
-Alert Logic Ciscomeraki AWS Based API Poll (PAWS) Log Collector Library.
+# Cisco Meraki collector
+Alert Logic Cisco Meraki AWS Based API Poll (PAWS) Log Collector Library.
 
 # Overview
 This repository contains the AWS JavaScript Lambda function and CloudFormation 
-Template (CFT) for deploying a log collector in AWS which will poll Ciscomeraki (Network Events) service API to collect and 
+Template (CFT) for deploying a log collector in AWS which will poll Cisco Meraki (Network Events) service API to collect and 
 forward logs to the Alert Logic CloudInsight backend services.
 
 # Installation
@@ -22,7 +22,7 @@ Instructions for setting up log collection from Cisco Meraki Dashboard using its
 1. **Enable API Access**:
    - Log in to your [Cisco Meraki Dashboard](https://dashboard.meraki.com) account.
    - You need to have access to organizational level administrative privileges.
-   -  Get the Organization ID at the bottom of the page ![ScreenShot](./docs/Ciscomerakiorg.png).
+   - Get the Organization ID at the bottom of the page ![ScreenShot](./docs/Ciscomerakiorg.png).
    - Navigate to *Organization > Settings*.
    - Under *Dashboard API access*, enable API access ![ScreenShot](./docs/Ciscomeraki_img1.png).
 
@@ -49,14 +49,14 @@ Instructions for setting up log collection from Cisco Meraki Dashboard using its
 - Throttling errors occur when your API requests exceed the allowed rate limit. When a throttling error occurs, the API will return an HTTP 429 status code along with an error message.
 - Refer to the [Throttling Errors documentation](https://developer.cisco.com/meraki/api/#/rest/guides/throttling-errors) for information on how to handle throttling errors and retry mechanisms.
 
-### 2. API Docs
+### API Docs
 
 1. [Network Events](https://developer.cisco.com/meraki/api-v1/get-network-events/)
 
    This endpoint allows you to retrieve network events from **all networks** within an organization, filtered by specific product types such as "appliance", "switch", and more. These events provide insights into network changes, device status updates, and other relevant activities.
 
 
-API URLs required for Ciscomeraki collector for Example
+API URLs required for Cisco Meraki collector for Example
 
 |     URL                              |
 |--------------------------------------|
@@ -145,5 +145,3 @@ make test
 make sam-local
 ```
   4. Please see `local/event.json` for the event payload used for local invocation.
-Please write your readme here
-

collectors/ciscomeraki/collector.js

@@ -14,32 +14,36 @@ const PawsCollector = require('@alertlogic/paws-collector').PawsCollector;
 const parse = require('@alertlogic/al-collector-js').Parse;
 const packageJson = require('./package.json');
 const calcNextCollectionInterval = require('@alertlogic/paws-collector').calcNextCollectionInterval;
-const utils = require("./utils");
+const merakiClient = require("./meraki_client");
 const AlLogger = require('@alertlogic/al-aws-collector-js').Logger;
 const MAX_POLL_INTERVAL = 900;
 const API_THROTTLING_ERROR = 429;
 const API_NOT_FOUND_ERROR = 404;
+const NOT_FOUND_ERROR_MAX_RETRIES = 3;
 
-let typeIdPaths = [{ path: ["type"] }];
-let tsPaths = [{ path: ["occurredAt"] }];
+const typeIdPaths = [{ path: ["type"] }];
+const tsPaths = [{ path: ["occurredAt"] }];
 
 class CiscomerakiCollector extends PawsCollector {
     constructor(context, creds) {
         super(context, creds, packageJson.version);
+        this.productTypes = process.env.paws_collector_param_string_1 ? JSON.parse(process.env.paws_collector_param_string_1) : [];
+        this.apiEndpoint = process.env.paws_endpoint.replace(/^https:\/\/|\/$/g, '');
+        this.orgKey = process.env.paws_collector_param_string_2;
     }
 
     async pawsInitCollectionState(event, callback) {
         let collector = this;
-        const resourceNames = process.env.collector_streams ? JSON.parse(process.env.collector_streams) : [];
         try {
+            const resourceNames = process.env.collector_streams ? JSON.parse(process.env.collector_streams) : [];
             if (resourceNames.length > 0) {
                 const initialStates = this.generateInitialStates(resourceNames);
                 return callback(null, initialStates, 1);
             }
             else {
                 try {
-                    const payloadObj = utils.getOrgKeySecretEndPoint(collector.secret);
-                    let networks = await utils.getAllNetworks(payloadObj);
+                    const payloadObj = merakiClient.getOrgKeySecretEndPoint(collector);
+                    let networks = await merakiClient.listNetworkIds(payloadObj);
                     if (networks.length > 0) {
                         const initialStates = this.generateInitialStates(networks);
                         return callback(null, initialStates, 1);
@@ -94,49 +98,54 @@ class CiscomerakiCollector extends PawsCollector {
     async handleUpdateStreamsFromNetworks() {
         let collector = this;
         try {
-            const payloadObj = utils.getOrgKeySecretEndPoint(collector.secret);
+            const payloadObj = merakiClient.getOrgKeySecretEndPoint(collector);
             //get networks from api
-            let networks = await utils.getAllNetworks(payloadObj);
+            let networks = await merakiClient.listNetworkIds(payloadObj);
             const keyValue = `${process.env.customer_id}/${collector._pawsCollectorType}/${collector.collector_id}/networks_${collector.collector_id}.json`;
-            let params = await utils.getS3ObjectParams(keyValue, undefined);
+            let params = await merakiClient.getS3ObjectParams(keyValue, undefined);
 
             //get networks json from s3 bucket
-            let networksFromS3 = await utils.fetchJsonFromS3Bucket(params.bucketName, params.key);
+            let networksFromS3 = await merakiClient.fetchJsonFromS3Bucket(params.bucketName, params.key);
             AlLogger.debug(`CMRI0000025 networks: ${JSON.stringify(networks)} networksFromS3 ${JSON.stringify(params)} ${JSON.stringify(networksFromS3)}`);
             if (networks.length > 0 && Array.isArray(networksFromS3) && networksFromS3.length > 0) {
-                let differenceNetworks = utils.differenceOfNetworksArray(networks, networksFromS3);
+                let differenceNetworks = merakiClient.differenceOfNetworksArray(networks, networksFromS3);
                 AlLogger.debug(`CMRI0000024 Networks updated ${JSON.stringify(differenceNetworks)}`);
 
                 if (differenceNetworks.length > 0) {
                     const initialStates = this.generateInitialStates(differenceNetworks);
                     AlLogger.debug(`CMRI0000020: SQS message added ${JSON.stringify(initialStates)}`);
                     collector._storeCollectionState({}, initialStates, this.pollInterval, async () => {
-                        await utils.uploadNetworksListInS3Bucket(keyValue, networks);
+                        await merakiClient.uploadNetworksListInS3Bucket(keyValue, networks);
                     });
                 } 
-            } else if (networksFromS3 && (networksFromS3.Code === 'NoSuchKey' || networksFromS3.Code === 'AccessDenied')) {
+            } else if (networksFromS3 && collector._isFileMissingError(networksFromS3.code)) {
                 AlLogger.debug(`CMRI0000026 networks ${JSON.stringify(params)} ${JSON.stringify(networks)}`);
-                await utils.uploadNetworksListInS3Bucket(keyValue, networks);
+                await merakiClient.uploadNetworksListInS3Bucket(keyValue, networks);
             }
         } catch (error) {
             AlLogger.debug(`Error updating streams from networks: ${error.message}`);
         }
     }
 
+    _isFileMissingError(code) { 
+        return code === 'NoSuchKey' || code === 'AccessDenied';
+    }
+
     pawsGetLogs(state, callback) {
         const collector = this;
 
-        const productTypes = process.env.paws_collector_param_string_1 ? JSON.parse(process.env.paws_collector_param_string_1) : [];
-        if (!productTypes) {
+        if (!collector.productTypes) {
             return callback("The Product Types was not found!");
         }
-        const { clientSecret, apiEndpoint, orgKey } = utils.getOrgKeySecretEndPoint(collector.secret, callback);
-        const apiDetails = utils.getAPIDetails(orgKey, productTypes);
+
+        const { clientSecret, apiEndpoint, orgKey } = merakiClient.getOrgKeySecretEndPoint(collector);
+
+        const apiDetails = merakiClient.getAPIDetails(orgKey, collector.productTypes);
         if (!apiDetails.url) {
             return callback("The API name was not found!");
         }
         AlLogger.info(`CMRI000001 Collecting data for NetworkId-${state.networkId} from ${state.since}`);
-        utils.getAPILogs(apiDetails, [], apiEndpoint, state, clientSecret, process.env.paws_max_pages_per_invocation)
+        merakiClient.getAPILogs(apiDetails, [], apiEndpoint, state, clientSecret, process.env.paws_max_pages_per_invocation)
             .then(({ accumulator, nextPage }) => {
                 let newState;
                 if (nextPage === undefined) {
@@ -171,7 +180,7 @@ class CiscomerakiCollector extends PawsCollector {
     handleOtherErrors(error, state, callback) {
         if (error && error.response && error.response.status == API_NOT_FOUND_ERROR) {
             state.retry = state.retry ? state.retry + 1 : 1;
-            if (state.retry > 3) {
+            if (state.retry > NOT_FOUND_ERROR_MAX_RETRIES) {
                 AlLogger.debug(`CMRI0000021 Deleted SQS message from Queue${JSON.stringify(state)}`);
                 this._invokeContext.succeed();
             } else {
@@ -217,7 +226,6 @@ class CiscomerakiCollector extends PawsCollector {
     }
 
     pawsFormatLog(msg) {
-        // TODO: double check that this message parsing fits your use case
         let collector = this;
 
         let ts = parse.getMsgTs(msg, tsPaths);

collectors/ciscomeraki/index.js

@@ -2,7 +2,7 @@
  * @copyright (C) 2019, Alert Logic, Inc
  * @doc
  *
- * Ciscomeraki System logs extension.
+ * Cisco Meraki System logs extension.
  *
  * @end
  * -----------------------------------------------------------------------------

collectors/ciscomeraki/local/sam-template.yaml

@@ -6,13 +6,12 @@ Resources:
   LocalLambda:
     Type: AWS::Serverless::Function
     Properties:
-      KmsKeyArn: arn:aws:kms:us-east-1:352283894008:key/c31cd559-a589-417b-91bb-19bfbcba903f
+      KmsKeyArn:
       Environment:
         Variables:
           AWS_LAMBDA_FUNCTION_NAME:
           aims_secret_key:
           LOG_LEVEL:
-          # DEBUG:
           aims_access_key_id:
           al_api:
           stack_name:
@@ -40,7 +39,7 @@ Resources:
           paws_poll_interval_delay:
           secret:
           dl_s3_bucket_name:
-      CodeUri: /Users/imransyed/Downloads/projects/may232024/paws-collector/collectors/ciscomeraki
+      CodeUri: s
       Runtime: nodejs18.x
       Handler: index.handler
       Timeout: 300 

collectors/ciscomeraki/utils.js → collectors/ciscomeraki/meraki_client.js

@@ -7,7 +7,7 @@ const AlAwsUtil = require('@alertlogic/al-aws-collector-js').Util;
 const NETWORKS_PER_PAGE = 1000;
 const EVENTS_PER_PAGE = 500;
 const API_THROTTLING_ERROR = 429;
-const DELAY_IN_SECS = 1000;
+const DEFAULT_RETRY_DELAY_MILLIS = 1000;
 
 async function getAPILogs(apiDetails, accumulator, apiEndpoint, state, clientSecret, maxPagesPerInvocation) {
     let nextPage;
@@ -42,7 +42,7 @@ async function getAPILogs(apiDetails, accumulator, apiEndpoint, state, clientSec
                     headers = response.headers;
                     const linkHeader = response.headers.link;
                     if (linkHeader && linkHeader.includes('rel=next')) {
-                        const nextLink = linkHeader.match(/<([^>]+)>; rel=next/)[1];
+                        const nextLink = linkHeader.match(/<(.*?)>; rel=next/)[1];
                         startingAfter = new URL(nextLink).searchParams.get('startingAfter');
                         since = startingAfter;
                         await getData(productType);
@@ -89,22 +89,22 @@ async function makeApiCall(url, apiKey, perPage, productType, startingAfter = nu
     }
 }
 
-async function getAllNetworks(payloadObj) {
-    const url = `/api/v1/organizations/${payloadObj.orgKey}/networks`;
+async function listNetworkIds(payloadObj) {
+    const resourcePath = `/api/v1/organizations/${payloadObj.orgKey}/networks`;
 
     try {
-        const networks = await fetchAllNetworks(url, payloadObj.clientSecret, payloadObj.apiEndpoint);
+        const networks = await fetchAllNetworks(resourcePath, payloadObj.clientSecret, payloadObj.apiEndpoint);
         return networks.map(network => network.id);
     } catch (error) {
         return error;
     }
 }
 
-async function fetchAllNetworks(url, apiKey, apiEndpoint) {
-    let delay = DELAY_IN_SECS; // Initial delay of 1 second
+async function fetchAllNetworks(resourcePath, apiKey, apiEndpoint) {
+    let delay = DEFAULT_RETRY_DELAY_MILLIS; // Initial delay of 1 second
     async function attemptApiCall() {
         try {
-            let response = await makeApiCall(`https://${apiEndpoint}/${url}`, apiKey, NETWORKS_PER_PAGE);
+            let response = await makeApiCall(`https://${apiEndpoint}/${resourcePath}`, apiKey, NETWORKS_PER_PAGE);
             return response.data;
         } catch (error) {
             if (error.response && error.response.status === API_THROTTLING_ERROR) {
@@ -120,18 +120,16 @@ async function fetchAllNetworks(url, apiKey, apiEndpoint) {
     return attemptApiCall();
 }
 
-function getOrgKeySecretEndPoint(secret) {
-    const clientSecret = secret;
+function getOrgKeySecretEndPoint(collector) {
+    const clientSecret = collector.secret;
     if (!clientSecret) {
         return "The Client Secret was not found!";
     }
-
-    const apiEndpoint = process.env.paws_endpoint.replace(/^https:\/\/|\/$/g, '');
-    const orgKey = process.env.paws_collector_param_string_2;
+    const apiEndpoint = collector.apiEndpoint;
+    const orgKey = collector.orgKey;
     if (!orgKey) {
         return "orgKey was not found!";
     }
-
     return { clientSecret, apiEndpoint, orgKey };
 }
 
@@ -210,9 +208,9 @@ module.exports = {
     getAPIDetails: getAPIDetails,
     makeApiCall: makeApiCall,
     getAPILogs: getAPILogs,
-    getAllNetworks: getAllNetworks,
-    fetchAllNetworks:fetchAllNetworks,
-    getOrgKeySecretEndPoint:getOrgKeySecretEndPoint,
+    listNetworkIds: listNetworkIds,
+    fetchAllNetworks: fetchAllNetworks,
+    getOrgKeySecretEndPoint: getOrgKeySecretEndPoint,
     uploadNetworksListInS3Bucket: uploadNetworksListInS3Bucket,
     getS3ObjectParams: getS3ObjectParams,
     uploadToS3Bucket: uploadToS3Bucket,

collectors/ciscomeraki/package.json

@@ -1,7 +1,7 @@
 {
   "name": "ciscomeraki-collector",
   "version": "1.0.0",
-  "description": "Alert Logic AWS based Ciscomeraki Log Collector",
+  "description": "Alert Logic AWS based Cisco Meraki Log Collector",
   "repository": {},
   "private": true,
   "scripts": {