Update Device Defender demo to send custom metrics (#1547)

Summary of changes:
Add support for collecting custom metrics for CPU usage time data (obtained from /proc/uptime) and System Memory data (obtained from /proc/meminfo)
Update utility for generating JSON report to support addition of custom metrics of string-list and number-list types for the 2 new metrics.
Update demo file to obtain the CPU usage time and memory data metrics from the support and add them to JSON report before sending to the Devide Defender service.
Update submodule for Device Defender library to access JSON report string literals relating to custom metrics.
Hygiene improvement of JSON report builder to use key names from Defender library instead of have them hard-coded in internal implementation so that either of short or long key names are used depending on user configuration of library.

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog for AWS IoT Device SDK for Embedded C
 
+## 202103.00 (March 2021)
+
+### Major Changes
+
+ - [#1547](https://github.com/aws/aws-iot-device-sdk-embedded-C/pull/1547) This release updates the Device Defender demo to showcase reporting of custom metrics to the AWS IoT Device Defender service.
+
 ## 202012.01 (December 2020)
 
 ### Major Changes

demos/defender/defender_demo_json/defender_demo.c

@@ -135,6 +135,12 @@ static uint16_t openUdpPorts[ OPEN_UDP_PORTS_ARRAY_SIZE ];
  */
 static Connection_t establishedConnections[ ESTABLISHED_CONNECTIONS_ARRAY_SIZE ];
 
+/**
+ * @brief Memory to represent custom metrics of CPU usage time and memory statistics
+ * that this demo sends to the AWS IoT Defender service.
+ */
+static CustomMetrics_t customMetrics;
+
 /**
  * @brief All the metrics sent in the device defender report.
  */
@@ -410,6 +416,32 @@ static bool collectDeviceMetrics( void )
         }
     }
 
+    /* Collect CPU usage time metrics from the system.
+     * This is an example of a custom metric of number-list type. */
+    if( metricsCollectorStatus == MetricsCollectorSuccess )
+    {
+        metricsCollectorStatus = GetCpuUsageStats( &( customMetrics.cpuUsageStats ) );
+
+        if( metricsCollectorStatus != MetricsCollectorSuccess )
+        {
+            LogError( ( "GetCpuUsageStats failed. Status: %d.",
+                        metricsCollectorStatus ) );
+        }
+    }
+
+    /* Collect metrics of memory statistics from the system.
+     * This is an example of a custom metric of string-list type. */
+    if( metricsCollectorStatus == MetricsCollectorSuccess )
+    {
+        metricsCollectorStatus = GetMemoryStats( &( customMetrics.memoryStats ) );
+
+        if( metricsCollectorStatus != MetricsCollectorSuccess )
+        {
+            LogError( ( "GetMemoryStats failed. Status: %d.",
+                        metricsCollectorStatus ) );
+        }
+    }
+
     /* Populate device metrics. */
     if( metricsCollectorStatus == MetricsCollectorSuccess )
     {
@@ -421,6 +453,7 @@ static bool collectDeviceMetrics( void )
         deviceMetrics.openUdpPortsArrayLength = numOpenUdpPorts;
         deviceMetrics.pEstablishedConnectionsArray = &( establishedConnections[ 0 ] );
         deviceMetrics.establishedConnectionsArrayLength = numEstablishedConnections;
+        deviceMetrics.pCustomMetrics = &( customMetrics );
     }
 
     return status;

demos/defender/defender_demo_json/demo_config.h

@@ -194,10 +194,8 @@
 
 /**
  * @brief Size of the buffer which contains the generated device defender report.
- *
- * If the generated report is larger than this, it is rejected.
  */
-#define DEVICE_METRICS_REPORT_BUFFER_SIZE      1000
+#define DEVICE_METRICS_REPORT_BUFFER_SIZE      2000
 
 /**
  * @brief Major version number of the device defender report.

demos/defender/defender_demo_json/metrics_collector.c

@@ -36,8 +36,8 @@
 #include "metrics_collector.h"
 
 /**
- * @brief The maximum length of line read from any of /proc/net/dev, /proc/net/tcp
- * and /proc/net/udp.
+ * @brief The maximum length of line read from any of /proc/net/dev, /proc/net/tcp,
+ * /proc/net/udp, /proc/uptime and /proc/meminfo files.
  */
 #define MAX_LINE_LENGTH                  ( 256 )
 
@@ -47,6 +47,12 @@
 #define CONNECTION_STATUS_LISTEN         ( 10 )
 #define CONNECTION_STATUS_ESTABLISHED    ( 1 )
 
+/**
+ * @brief Fields from /proc/meminfo to use for memory statistics.
+ */
+#define TOTAL_MEM_FIELD                  "MemTotal"
+#define AVAILABLE_MEM_FIELD              "MemAvailable"
+
 /**
  * @brief Get a list of the open ports.
  *
@@ -378,4 +384,155 @@ MetricsCollectorStatus_t GetEstablishedConnections( Connection_t * pOutConnectio
 
     return status;
 }
+
 /*-----------------------------------------------------------*/
+
+MetricsCollectorStatus_t GetCpuUsageStats( CpuUsageStats_t * pCpuUsage )
+{
+    MetricsCollectorStatus_t status = MetricsCollectorSuccess;
+    FILE * fileHandle = NULL;
+    uint32_t filledVariables;
+    char lineBuffer[ MAX_LINE_LENGTH ];
+
+    if( pCpuUsage == NULL )
+    {
+        LogError( ( "Invalid parameter. pCpuUsage: %p", ( void * ) pCpuUsage ) );
+        status = MetricsCollectorBadParameter;
+    }
+
+    if( status == MetricsCollectorSuccess )
+    {
+        fileHandle = fopen( "/proc/uptime", "r" );
+
+        if( fileHandle == NULL )
+        {
+            LogError( ( "Failed to open /proc/uptime." ) );
+            status = MetricsCollectorFileOpenFailed;
+        }
+    }
+
+    if( status == MetricsCollectorSuccess )
+    {
+        if( fgets( &( lineBuffer[ 0 ] ), MAX_LINE_LENGTH, fileHandle ) != NULL )
+        {
+            LogDebug( ( "File: /proc/uptime, Content: %s.",
+                        &( lineBuffer[ 0 ] ) ) );
+
+            /* Parse the output. */
+            filledVariables = sscanf( &( lineBuffer[ 0 ] ),
+                                      "%u.%*u %u.%*u",
+                                      &( pCpuUsage->upTime ),
+                                      &( pCpuUsage->idleTime ) );
+
+            /* sscanf should fill all the 2 variables successfully. */
+            if( filledVariables != 2 )
+            {
+                LogError( ( "Failed to parse CPU usage data. File: /proc/uptime, Data: %s.", &( lineBuffer[ 0 ] ) ) );
+                status = MetricsCollectorParsingFailed;
+            }
+        }
+    }
+
+    if( fileHandle != NULL )
+    {
+        fclose( fileHandle );
+    }
+
+    return status;
+}
+/*-----------------------------------------------------------*/
+
+MetricsCollectorStatus_t GetMemoryStats( MemoryStats_t * pMemoryStats )
+{
+    MetricsCollectorStatus_t status = MetricsCollectorSuccess;
+    FILE * fileHandle = NULL;
+    /* Variables for reading and processing data from "/proc/meminfo" file. */
+    char lineBuffer[ MAX_LINE_LENGTH ];
+    bool readTotalMem = false, readAvailableMem = false;
+    int filledVariables = 0;
+
+    if( ( pMemoryStats == NULL ) )
+    {
+        LogError( ( "Invalid parameter. pMemoryStats: %p", ( void * ) pMemoryStats ) );
+        status = MetricsCollectorBadParameter;
+    }
+
+    if( status == MetricsCollectorSuccess )
+    {
+        fileHandle = fopen( "/proc/meminfo", "r" );
+
+        if( fileHandle == NULL )
+        {
+            LogError( ( "Failed to open /proc/meminfo." ) );
+            status = MetricsCollectorFileOpenFailed;
+        }
+    }
+
+    if( status == MetricsCollectorSuccess )
+    {
+        while( ( fgets( &( lineBuffer[ 0 ] ), MAX_LINE_LENGTH, fileHandle ) != NULL ) )
+        {
+            LogDebug( ( "File: /proc/meminfo, Content: %s.", &( lineBuffer[ 0 ] ) ) );
+
+            /* Check if the line read represents information for total memory in the system. */
+            if( strncmp( lineBuffer, TOTAL_MEM_FIELD, ( sizeof( TOTAL_MEM_FIELD ) - 1UL ) ) == 0 )
+            {
+                /* Extract the total memory value from the line. */
+                filledVariables = sscanf( lineBuffer,
+                                          "%*[^1-9]%u kB",
+                                          ( &pMemoryStats->totalMemory ) );
+
+                if( filledVariables != 1 )
+                {
+                    LogError( ( "Failed to parse data. File: /proc/meminfo, Content: %s", lineBuffer ) );
+                    status = MetricsCollectorParsingFailed;
+
+                    break;
+                }
+                else
+                {
+                    readTotalMem = true;
+                }
+            }
+            /* Check if the line read represents information for available memory in the system. */
+            else if( strncmp( lineBuffer, AVAILABLE_MEM_FIELD, ( sizeof( AVAILABLE_MEM_FIELD ) - 1UL ) ) == 0 )
+            {
+                /* Extract the total memory value from the line. */
+                filledVariables = sscanf( lineBuffer,
+                                          "%*[^1-9]%u kB",
+                                          ( &pMemoryStats->availableMemory ) );
+
+                if( filledVariables != 1 )
+                {
+                    LogError( ( "Failed to parse data. File: /proc/meminfo, Content: %s", lineBuffer ) );
+                    status = MetricsCollectorParsingFailed;
+
+                    break;
+                }
+                else
+                {
+                    readAvailableMem = true;
+                }
+            }
+
+            if( readTotalMem && readAvailableMem )
+            {
+                /* We have obtained data for both total and available memory in the system. */
+                status = MetricsCollectorSuccess;
+
+                break;
+            }
+            else
+            {
+                status = MetricsCollectorDataNotFound;
+            }
+        }
+    }
+
+    if( fileHandle != NULL )
+    {
+        fclose( fileHandle );
+    }
+
+    return status;
+}

demos/defender/defender_demo_json/metrics_collector.h

@@ -34,7 +34,8 @@ typedef enum
     MetricsCollectorSuccess = 0,
     MetricsCollectorBadParameter,
     MetricsCollectorFileOpenFailed,
-    MetricsCollectorParsingFailed
+    MetricsCollectorParsingFailed,
+    MetricsCollectorDataNotFound,
 } MetricsCollectorStatus_t;
 
 /**
@@ -59,6 +60,28 @@ typedef struct Connection
     uint16_t remotePort;
 } Connection_t;
 
+/**
+ * @brief Represents the Cpu Usage statistics obtained from "/proc/uptime".
+ * Refer to Linux manual for "/proc" filesystem for more information.
+ * https://man7.org/linux/man-pages/man5/procfs.5.html
+ */
+typedef struct CpuUsageData
+{
+    uint32_t upTime;   /**< Up-time of system in seconds. */
+    uint32_t idleTime; /**< Idle time of system in seconds. */
+} CpuUsageStats_t;
+
+/**
+ * @brief Represents the memory data of total and available memory from "/proc/uptime".
+ * Refer to Linux manual for "/proc" filesystem for more information.
+ * https://man7.org/linux/man-pages/man5/procfs.5.html
+ */
+typedef struct MemoryStats
+{
+    uint32_t totalMemory;     /**< Amount of total memory in system (in kB). */
+    uint32_t availableMemory; /**< Amount of available memory in system (in kB). */
+} MemoryStats_t;
+
 /**
  * @brief Get network stats.
  *
@@ -145,4 +168,34 @@ MetricsCollectorStatus_t GetEstablishedConnections( Connection_t * pOutConnectio
                                                     uint32_t connectionsArrayLength,
                                                     uint32_t * pOutNumEstablishedConnections );
 
+/**
+ * @brief Get CPU usage data of uptime and idle time from the system.
+ *
+ * This function finds the system CPU information by reading the "/proc/uptime" file.
+ *
+ * @param[out] pCpuUsage The memory to write the CPU usage statistics into.
+ *
+ * @return #MetricsCollectorSuccess if CPU usage data is successfully obtained;
+ * #MetricsCollectorBadParameter if invalid parameter is passed;
+ * #MetricsCollectorFileOpenFailed if the function fails to open "/proc/uptime";
+ * MetricsCollectorParsingFailed if the function fails to parses the data read
+ * from "/proc/uptime".
+ */
+MetricsCollectorStatus_t GetCpuUsageStats( CpuUsageStats_t * pCpuUsage );
+
+/**
+ * @brief Gets data of total and available memory from the system.
+ *
+ * This function finds the memory information by reading the "/proc/meminfo" file.
+ *
+ * @param[out] pMemoryStats The memory to write the memory information into.
+ *
+ * @return #MetricsCollectorSuccess if memory data statistic is successfully calculated;
+ * #MetricsCollectorBadParameter if invalid parameter is passed;
+ * #MetricsCollectorFileOpenFailed if the function fails to open "/proc/meminfo";
+ * MetricsCollectorParsingFailed if the function fails to parses the data read
+ * from "/proc/meminfo".
+ */
+MetricsCollectorStatus_t GetMemoryStats( MemoryStats_t * pMemoryStats );
+
 #endif /* ifndef METRICS_COLLECTOR_H_ */