Run performance e2e as part of build (runs when ctest is called)

Actually add run-e2e tests in linux jenkins build.

Add verbosity to ctest

E2E tests now can find their modules on Release builds, too.

core/tests/gateway_e2e/module_config_windows.c

@@ -1,11 +1,17 @@
-// Copyright (c) Microsoft. All rights reserved.
+// Copyright (c) Microsoft. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
 #include "gateway.h"
 
+#ifdef _DEBUG
 const char* e2e_module_path_string = "..\\e2e_module\\Debug\\e2e_module.dll";
 const char* iothub_path_string = "..\\..\\..\\..\\modules\\iothub\\Debug\\iothub.dll";
 const char* identity_map_path_string = "..\\..\\..\\..\\modules\\identitymap\\Debug\\identity_map.dll";
+#else
+const char* e2e_module_path_string = "..\\e2e_module\\Release\\e2e_module.dll";
+const char* iothub_path_string = "..\\..\\..\\..\\modules\\iothub\\Release\\iothub.dll";
+const char* identity_map_path_string = "..\\..\\..\\..\\modules\\identitymap\\Release\\identity_map.dll"; 
+#endif
 
 const char* e2e_module_path()
 {

core/tests/performance_e2e/CMakeLists.txt

@@ -60,6 +60,7 @@ if(install_modules)
 endif()
 
 
+# This builds the command line tool.
 set(performance_e2e_sources
     ./src/main.cpp
 )
@@ -89,3 +90,65 @@ copy_gateway_dll(performance_e2e ${CMAKE_CURRENT_BINARY_DIR}/$(Configuration) )
 set_target_properties(performance_e2e
             PROPERTIES
             FOLDER "tests/E2ETests")
+
+# Run E2E as a test.
+
+set(theseTestsName performance_e2e)
+
+#Disable valgrind just for performance E2E test 
+set(run_valgrind OFF)
+
+compileAsC99()
+
+set(${theseTestsName}_test_files
+    ./src/${theseTestsName}.c
+)
+
+#setting the dynamic_loader file based on OS that it is used
+if(WIN32)
+    set(modules_c_file ./src/module_config_windows.c)
+elseif(UNIX) # LINUX OR APPLE
+    set(modules_c_file ./src/module_config_linux.c)
+endif()
+
+
+set(${theseTestsName}_c_files
+    ${modules_c_file}
+)
+
+set(${theseTestsName}_h_files
+    ./inc/module_config_resources.h
+)
+
+build_c_test_artifacts(${theseTestsName} ON "tests/E2ETests")
+set_target_properties(
+    ${theseTestsName}_exe PROPERTIES
+    BUILD_WITH_INSTALL_RPATH TRUE
+)
+
+linkSharedUtil(${theseTestsName}_exe)
+
+if(WIN32)
+    if(TARGET ${theseTestsName}_dll)
+        target_link_libraries(${theseTestsName}_dll
+            gateway
+        )
+    endif()
+
+    if(TARGET ${theseTestsName}_exe)
+        target_link_libraries(${theseTestsName}_exe
+            gateway
+        )    
+        install_broker(${theseTestsName}_exe ${CMAKE_CURRENT_BINARY_DIR}/$(Configuration) )
+        copy_gateway_dll(${theseTestsName}_exe ${CMAKE_CURRENT_BINARY_DIR}/$(Configuration) )
+    endif()
+else()
+    if(TARGET ${theseTestsName}_exe)
+        target_link_libraries(${theseTestsName}_exe
+            gateway
+            aziotsharedutil
+            pthread
+        )
+    endif()
+
+endif()

core/tests/performance_e2e/README.md

@@ -386,9 +386,14 @@ resources allocated in `moduleHandle`.
 
 ## Running the performance test. 
 
+To run on the command line, use the `performance_e2e` executable.
+
 The program accepts 2 command line arguments, the json configuration file and an 
 optional test duration in seconds. The sample creates the gateway from the JSON 
 file, starts all modules, pauses for a given time (default of 5 seconds), and 
 stops the gateway. Stopping the gateway will trigger the metrics module to 
 report message statistics.
 
+A 5 second and 10 second performance test are run as part of the build tests.
+run `ctest -C Debug -V -R performance_e2e` to execute those tests.
+

core/tests/performance_e2e/inc/module_config_resources.h

@@ -0,0 +1,18 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+#ifndef MODULE_CONFIG_RESOURCES_H
+#define MODULE_CONFIG_RESOURCES_H
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+//Add here paths for any new Module. 
+const char* simulator_module_path();
+const char* metrics_module_path();
+#ifdef __cplusplus
+}
+#endif
+
+#endif /*MODULE_CONFIG_RESOURCES_H*/

core/tests/performance_e2e/main.c

@@ -0,0 +1,11 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+#include "testrunnerswitcher.h"
+
+int main(void)
+{
+    size_t failedTestCount = 0;
+    RUN_TEST_SUITE(Performance_e2e, failedTestCount);
+    return failedTestCount;
+}

core/tests/performance_e2e/src/metrics.cpp

@@ -241,6 +241,7 @@ static void MetricsModule_Destroy(MODULE_HANDLE moduleHandle)
                     << "Messages Lost: " << (*d).second.messages_lost << std::endl;
             }
         }    
+        delete (module->per_device_metrics);
         free(module);
     }
 }

core/tests/performance_e2e/src/module_config_linux.c

@@ -0,0 +1,15 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+const char* simulator_module_path_string = "libsimulator.so";
+const char* metrics_module_path_string = "libmetrics.so";
+
+const char* simulator_module_path()
+{
+    return simulator_module_path_string;
+}
+
+const char* metrics_module_path()
+{
+    return metrics_module_path_string;
+}

core/tests/performance_e2e/src/module_config_windows.c

@@ -0,0 +1,22 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+#include "gateway.h"
+
+#ifdef _DEBUG
+const char* simulator_module_path_string = "Debug\\simulator.dll";
+const char* metrics_module_path_string = "Debug\\metrics.dll";
+#else
+const char* simulator_module_path_string = "Release\\simulator.dll";
+const char* metrics_module_path_string = "Release\\metrics.dll";
+#endif
+
+const char* simulator_module_path()
+{
+    return simulator_module_path_string;
+}
+
+const char* metrics_module_path()
+{
+    return metrics_module_path_string;
+}

core/tests/performance_e2e/src/performance_e2e.c

@@ -0,0 +1,192 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+#include "gateway.h"
+#include "module_config_resources.h"
+#include "simulator.h"
+#include "module_loader.h"
+#include "module_loaders/dynamic_loader.h"
+#include "azure_c_shared_utility/threadapi.h"
+
+#include "testrunnerswitcher.h"
+
+//=============================================================================
+//Globals
+//=============================================================================
+
+#ifdef WIN32
+static TEST_MUTEX_HANDLE g_dllByDll;
+#endif
+static TEST_MUTEX_HANDLE g_testByTest;
+
+BEGIN_TEST_SUITE(Performance_e2e)
+
+TEST_SUITE_INITIALIZE(TestClassInitialize)
+{
+    TEST_INITIALIZE_MEMORY_DEBUG(g_dllByDll);
+    g_testByTest = TEST_MUTEX_CREATE();
+    ASSERT_IS_NOT_NULL(g_testByTest);
+
+}
+
+TEST_SUITE_CLEANUP(TestClassCleanup)
+{
+    TEST_MUTEX_DESTROY(g_testByTest);
+    TEST_DEINITIALIZE_MEMORY_DEBUG(g_dllByDll);
+}
+
+TEST_FUNCTION_INITIALIZE(TestMethodInitialize)
+{
+    if (TEST_MUTEX_ACQUIRE(g_testByTest) != 0)
+    {
+        ASSERT_FAIL("our mutex is ABANDONED. Failure in test framework");
+    }
+
+}
+
+TEST_FUNCTION_CLEANUP(TestMethodCleanup)
+{
+    TEST_MUTEX_RELEASE(g_testByTest);
+}
+
+TEST_FUNCTION(Performance_e2e_5_second_run)
+{
+        ///arrange
+        GATEWAY_HANDLE e2eGatewayInstance;
+
+        /* Setup: data for simulator module */
+        SIMULATOR_MODULE_CONFIG simulator_config = 
+        {
+            "device1",
+            0,
+            2,
+            16,
+            256
+        };
+
+        GATEWAY_MODULES_ENTRY modules[2];
+		DYNAMIC_LOADER_ENTRYPOINT loader_info[2];
+        GATEWAY_LINK_ENTRY links[1];
+
+        // simulator
+		modules[0].module_name = "simulator1";
+        modules[0].module_configuration = &simulator_config;
+		modules[0].module_loader_info.loader = DynamicLoader_Get();
+		loader_info[0].moduleLibraryFileName = STRING_construct(simulator_module_path());
+		modules[0].module_loader_info.entrypoint = (void*)&(loader_info[0]);
+
+        // metrics
+		modules[1].module_name = "metrics1";
+		modules[1].module_configuration = NULL;
+		modules[1].module_loader_info.loader = DynamicLoader_Get();
+		loader_info[1].moduleLibraryFileName = STRING_construct(metrics_module_path());
+		modules[1].module_loader_info.entrypoint = (void*)&(loader_info[1]);
+
+        links[0].module_source = "simulator1";
+        links[0].module_sink = "metrics1";
+
+        GATEWAY_PROPERTIES performance_gw_properties;
+        VECTOR_HANDLE gatewayProps = VECTOR_create(sizeof(GATEWAY_MODULES_ENTRY));
+        VECTOR_HANDLE gatewayLinks = VECTOR_create(sizeof(GATEWAY_LINK_ENTRY));
+
+        VECTOR_push_back(gatewayProps, &modules, 2);
+        VECTOR_push_back(gatewayLinks, &links, 1);
+
+
+
+        ///act
+        performance_gw_properties.gateway_modules = gatewayProps;
+        performance_gw_properties.gateway_links = gatewayLinks; 
+        e2eGatewayInstance = Gateway_Create(&performance_gw_properties);
+        GATEWAY_START_RESULT start_result = Gateway_Start(e2eGatewayInstance);
+
+        ///assert
+        ASSERT_IS_NOT_NULL(e2eGatewayInstance);
+        ASSERT_IS_TRUE((start_result == GATEWAY_START_SUCCESS));
+
+
+        ThreadAPI_Sleep(5000);
+
+        Gateway_Destroy(e2eGatewayInstance);
+
+        VECTOR_destroy(gatewayProps);
+        VECTOR_destroy(gatewayLinks);
+
+		for (int loader = 0; loader < 2; loader++)
+		{
+			STRING_delete(loader_info[loader].moduleLibraryFileName);
+		}
+}
+
+TEST_FUNCTION(Performance_e2e_10_second_run)
+{
+        ///arrange
+        GATEWAY_HANDLE e2eGatewayInstance;
+
+        /* Setup: data for simulator module */
+        SIMULATOR_MODULE_CONFIG simulator_config = 
+        {
+            "device1",
+            0,
+            2,
+            16,
+            256
+        };
+
+        GATEWAY_MODULES_ENTRY modules[2];
+		DYNAMIC_LOADER_ENTRYPOINT loader_info[2];
+        GATEWAY_LINK_ENTRY links[1];
+
+        // simulator
+		modules[0].module_name = "simulator1";
+        modules[0].module_configuration = &simulator_config;
+		modules[0].module_loader_info.loader = DynamicLoader_Get();
+		loader_info[0].moduleLibraryFileName = STRING_construct(simulator_module_path());
+		modules[0].module_loader_info.entrypoint = (void*)&(loader_info[0]);
+
+        // metrics
+		modules[1].module_name = "metrics1";
+		modules[1].module_configuration = NULL;
+		modules[1].module_loader_info.loader = DynamicLoader_Get();
+		loader_info[1].moduleLibraryFileName = STRING_construct(metrics_module_path());
+		modules[1].module_loader_info.entrypoint = (void*)&(loader_info[1]);
+
+        links[0].module_source = "simulator1";
+        links[0].module_sink = "metrics1";
+
+        GATEWAY_PROPERTIES performance_gw_properties;
+        VECTOR_HANDLE gatewayProps = VECTOR_create(sizeof(GATEWAY_MODULES_ENTRY));
+        VECTOR_HANDLE gatewayLinks = VECTOR_create(sizeof(GATEWAY_LINK_ENTRY));
+
+        VECTOR_push_back(gatewayProps, &modules, 2);
+        VECTOR_push_back(gatewayLinks, &links, 1);
+
+
+
+        ///act
+        performance_gw_properties.gateway_modules = gatewayProps;
+        performance_gw_properties.gateway_links = gatewayLinks; 
+        e2eGatewayInstance = Gateway_Create(&performance_gw_properties);
+        GATEWAY_START_RESULT start_result = Gateway_Start(e2eGatewayInstance);
+
+        ///assert
+        ASSERT_IS_NOT_NULL(e2eGatewayInstance);
+        ASSERT_IS_TRUE((start_result == GATEWAY_START_SUCCESS));
+
+
+        ThreadAPI_Sleep(10000);
+
+        Gateway_Destroy(e2eGatewayInstance);
+
+        VECTOR_destroy(gatewayProps);
+        VECTOR_destroy(gatewayLinks);
+
+		for (int loader = 0; loader < 2; loader++)
+		{
+			STRING_delete(loader_info[loader].moduleLibraryFileName);
+		}
+
+}
+
+
+END_TEST_SUITE(Performance_e2e);