Kv perf pipeline (Azure#4222)

* secrets * perftestst for get Secret * dss * qfe * pipelines * comments * cleanup script * comment * key last * somemore stuff * clang
LarryOsterman · Jan 17, 2023 · b7e7e4e · b7e7e4e
1 parent 4f7a0ea
commit b7e7e4e
Show file tree

Hide file tree

Showing 11 changed files with 453 additions and 3 deletions.
diff --git a/CMakeSettings.json b/CMakeSettings.json
@@ -7,7 +7,7 @@
       "inheritEnvironments": [ "msvc_x64_x64" ],
       "buildRoot": "${projectDir}\\out\\build\\${name}",
       "installRoot": "${projectDir}\\out\\install\\${name}",
-      "cmakeCommandArgs": "-DINSTALL_GTEST=OFF -DBUILD_TESTING=ON -DBUILD_TRANSPORT_CURL=ON -DBUILD_SAMPLES=ON",
+      "cmakeCommandArgs": "-DINSTALL_GTEST=OFF -DBUILD_TESTING=ON -DBUILD_TRANSPORT_CURL=ON -DBUILD_SAMPLES=ON -DBUILD_PERFORMANCE_TESTS=ON",
       "buildCommandArgs": "-v",
       "ctestCommandArgs": "",
       "variables": [

diff --git a/sdk/keyvault/azure-security-keyvault-certificates/perf.yml b/sdk/keyvault/azure-security-keyvault-certificates/perf.yml
@@ -0,0 +1,40 @@
+parameters:
+- name: PackageVersions
+  displayName: PackageVersions (regex of package versions to run)
+  type: string
+  default: '1|source'
+- name: Tests
+  displayName: Tests (regex of tests to run)
+  type: string
+  default: '^(get-certificate)$'
+- name: Arguments
+  displayName: Arguments (regex of arguments to run)
+  type: string
+  default: '.*'
+- name: Iterations
+  displayName: Iterations (times to run each test)
+  type: number
+  default: '5'
+- name: AdditionalArguments
+  displayName: AdditionalArguments (passed to PerfAutomation)
+  type: string 
+  default: ' '
+
+extends:
+  template: /eng/pipelines/templates/jobs/perf.yml
+  parameters:
+    ServiceDirectory: keyvault
+    Services: "^keyvault-certificates$"
+    PackageVersions: ${{ parameters.PackageVersions }}
+    Tests: ${{ parameters.Tests }}
+    Arguments: ${{ parameters.Arguments }}
+    Iterations: ${{ parameters.Iterations }}
+    AdditionalArguments: ${{ parameters.AdditionalArguments }}
+    InstallLanguageSteps: 
+      - pwsh: |
+          Write-Host "##vso[task.setvariable variable=VCPKG_BINARY_SOURCES_SECRET;issecret=true;]clear;x-azblob,https://cppvcpkgcache.blob.core.windows.net/public-vcpkg-container,,read"
+        displayName: Set Vcpkg Variables
+
+    EnvVars: 
+      # This is set in the InstallLanguageSteps 
+      VCPKG_BINARY_SOURCES_SECRET: $(VCPKG_BINARY_SOURCES_SECRET)
diff --git a/...ault-certificates/test/perf/inc/azure/keyvault/certificates/test/get_certificate_test.hpp b/...ault-certificates/test/perf/inc/azure/keyvault/certificates/test/get_certificate_test.hpp
@@ -28,7 +28,7 @@ namespace Azure {
         namespace Test {
 
   /**
-   * @brief A test to measure getting a key performance.
+   * @brief A test to measure getting a certificate performance.
    *
    */
   class GetCertificate : public Azure::Perf::PerfTest {

diff --git a/sdk/keyvault/azure-security-keyvault-keys/perf.yml b/sdk/keyvault/azure-security-keyvault-keys/perf.yml
@@ -0,0 +1,40 @@
+parameters:
+- name: PackageVersions
+  displayName: PackageVersions (regex of package versions to run)
+  type: string
+  default: '1|source'
+- name: Tests
+  displayName: Tests (regex of tests to run)
+  type: string
+  default: '^(get-key)$'
+- name: Arguments
+  displayName: Arguments (regex of arguments to run)
+  type: string
+  default: '.*'
+- name: Iterations
+  displayName: Iterations (times to run each test)
+  type: number
+  default: '5'
+- name: AdditionalArguments
+  displayName: AdditionalArguments (passed to PerfAutomation)
+  type: string 
+  default: ' '
+
+extends:
+  template: /eng/pipelines/templates/jobs/perf.yml
+  parameters:
+    ServiceDirectory: keyvault
+    Services: "^keyvault-keys$"
+    PackageVersions: ${{ parameters.PackageVersions }}
+    Tests: ${{ parameters.Tests }}
+    Arguments: ${{ parameters.Arguments }}
+    Iterations: ${{ parameters.Iterations }}
+    AdditionalArguments: ${{ parameters.AdditionalArguments }}
+    InstallLanguageSteps: 
+      - pwsh: |
+          Write-Host "##vso[task.setvariable variable=VCPKG_BINARY_SOURCES_SECRET;issecret=true;]clear;x-azblob,https://cppvcpkgcache.blob.core.windows.net/public-vcpkg-container,,read"
+        displayName: Set Vcpkg Variables
+
+    EnvVars: 
+      # This is set in the InstallLanguageSteps 
+      VCPKG_BINARY_SOURCES_SECRET: $(VCPKG_BINARY_SOURCES_SECRET)
diff --git a/sdk/keyvault/azure-security-keyvault-secrets/CMakeLists.txt b/sdk/keyvault/azure-security-keyvault-secrets/CMakeLists.txt
@@ -114,7 +114,7 @@ if(BUILD_TESTING)
 endif()
 
 if (BUILD_PERFORMANCE_TESTS)
-  #add_subdirectory(test/perf)
+  add_subdirectory(test/perf)
 endif()
 
 if(BUILD_SAMPLES)

diff --git a/sdk/keyvault/azure-security-keyvault-secrets/perf.yml b/sdk/keyvault/azure-security-keyvault-secrets/perf.yml
@@ -0,0 +1,40 @@
+parameters:
+- name: PackageVersions
+  displayName: PackageVersions (regex of package versions to run)
+  type: string
+  default: '1|source'
+- name: Tests
+  displayName: Tests (regex of tests to run)
+  type: string
+  default: '^(get-secret|list-secrets)$'
+- name: Arguments
+  displayName: Arguments (regex of arguments to run)
+  type: string
+  default: '.*'
+- name: Iterations
+  displayName: Iterations (times to run each test)
+  type: number
+  default: '5'
+- name: AdditionalArguments
+  displayName: AdditionalArguments (passed to PerfAutomation)
+  type: string 
+  default: ' '
+
+extends:
+  template: /eng/pipelines/templates/jobs/perf.yml
+  parameters:
+    ServiceDirectory: keyvault
+    Services: "^keyvault-secrets$"
+    PackageVersions: ${{ parameters.PackageVersions }}
+    Tests: ${{ parameters.Tests }}
+    Arguments: ${{ parameters.Arguments }}
+    Iterations: ${{ parameters.Iterations }}
+    AdditionalArguments: ${{ parameters.AdditionalArguments }}
+    InstallLanguageSteps: 
+      - pwsh: |
+          Write-Host "##vso[task.setvariable variable=VCPKG_BINARY_SOURCES_SECRET;issecret=true;]clear;x-azblob,https://cppvcpkgcache.blob.core.windows.net/public-vcpkg-container,,read"
+        displayName: Set Vcpkg Variables
+
+    EnvVars: 
+      # This is set in the InstallLanguageSteps 
+      VCPKG_BINARY_SOURCES_SECRET: $(VCPKG_BINARY_SOURCES_SECRET)
diff --git a/sdk/keyvault/azure-security-keyvault-secrets/test/perf/CMakeLists.txt b/sdk/keyvault/azure-security-keyvault-secrets/test/perf/CMakeLists.txt
@@ -0,0 +1,38 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# SPDX-License-Identifier: MIT
+
+# Configure CMake project.
+cmake_minimum_required (VERSION 3.13)
+project(azure-security-keyvault-secrets-perf LANGUAGES CXX)
+set(CMAKE_CXX_STANDARD 14)
+set(CMAKE_CXX_STANDARD_REQUIRED True)
+
+set(
+  AZURE_KEYVAULT_SECRETS_PERF_TEST_HEADER
+  inc/azure/keyvault/secrets/test/get_secret_test.hpp
+)
+
+set(
+  AZURE_KEYVAULT_SECRETS_PERF_TEST_SOURCE
+    src/azure_security_keyvault_secrets_perf_test.cpp
+)
+
+# Name the binary to be created.
+add_executable (
+  azure-security-keyvault-secrets-perf
+     ${AZURE_KEYVAULT_SECRETS_PERF_TEST_HEADER} ${AZURE_KEYVAULT_SECRETS_PERF_TEST_SOURCE}
+)
+create_per_service_target_build(keyvault azure-security-keyvault-secrets-perf)
+create_map_file(azure-security-keyvault-secrets-perf azure-security-keyvault-secrets-perf.map)
+
+# Include the headers from the project.
+target_include_directories(
+  azure-security-keyvault-secrets-perf
+    PUBLIC
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/inc>
+)
+
+# link the `azure-perf` lib together with any other library which will be used for the tests. 
+target_link_libraries(azure-security-keyvault-secrets-perf PRIVATE azure-identity azure-security-keyvault-secrets azure-perf)
+# Make sure the project will appear in the test folder for Visual Studio CMake view
+set_target_properties(azure-security-keyvault-secrets-perf PROPERTIES FOLDER "Tests/Keyvault")
diff --git a/...e-security-keyvault-secrets/test/perf/inc/azure/keyvault/secrets/test/get_secret_test.hpp b/...e-security-keyvault-secrets/test/perf/inc/azure/keyvault/secrets/test/get_secret_test.hpp
@@ -0,0 +1,126 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+/**
+ * @file
+ * @brief Test the overhead of getting a secret.
+ *
+ */
+
+#pragma once
+
+#include <azure/perf.hpp>
+
+#include <azure/core/internal/environment.hpp>
+#include <azure/identity.hpp>
+#include <azure/keyvault/secrets.hpp>
+
+#include <memory>
+#include <string>
+#include <vector>
+
+using namespace Azure::Core::_internal;
+namespace Azure { namespace Security { namespace KeyVault { namespace Secrets { namespace Test {
+
+  /**
+   * @brief A test to measure getting a secret performance.
+   *
+   */
+  class GetSecret : public Azure::Perf::PerfTest {
+  private:
+    std::string m_vaultUrl;
+    std::string m_secretName;
+    std::string m_tenantId;
+    std::string m_clientId;
+    std::string m_secret;
+    std::shared_ptr<Azure::Identity::ClientSecretCredential> m_credential;
+    std::unique_ptr<Azure::Security::KeyVault::Secrets::SecretClient> m_client;
+
+  public:
+    /**
+     * @brief Get the Ids and secret
+     *
+     */
+    void Setup() override
+    {
+      m_vaultUrl = m_options.GetOptionOrDefault<std::string>(
+          "vaultUrl", Environment::GetVariable("AZURE_KEYVAULT_URL"));
+      m_tenantId = m_options.GetOptionOrDefault<std::string>(
+          "TenantId", Environment::GetVariable("AZURE_TENANT_ID"));
+      m_clientId = m_options.GetOptionOrDefault<std::string>(
+          "ClientId", Environment::GetVariable("AZURE_CLIENT_ID"));
+      m_secret = m_options.GetOptionOrDefault<std::string>(
+          "Secret", Environment::GetVariable("AZURE_CLIENT_SECRET"));
+      m_credential = std::make_shared<Azure::Identity::ClientSecretCredential>(
+          m_tenantId, m_clientId, m_secret);
+      m_client = std::make_unique<Azure::Security::KeyVault::Secrets::SecretClient>(
+          m_vaultUrl,
+          m_credential,
+          InitClientOptions<Azure::Security::KeyVault::Secrets::SecretClientOptions>());
+      this->CreateRandomNameKey();
+    }
+
+    /**
+     * @brief Create a random named secret.
+     *
+     */
+    void CreateRandomNameKey()
+    {
+      std::string name("perf");
+      int suffixLen = 10;
+      static const char alphanum[]
+          = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
+      std::string suffix;
+      suffix.reserve(suffixLen);
+
+      for (int i = 0; i < suffixLen; ++i)
+      {
+        suffix += alphanum[rand() % (sizeof(alphanum) - 1)];
+      }
+
+      m_secretName = name + suffix;
+
+      auto secretResponse = m_client->SetSecret(m_secretName, "secretValue");
+    }
+
+    /**
+     * @brief Construct a new GetSecret test.
+     *
+     * @param options The test options.
+     */
+    GetSecret(Azure::Perf::TestOptions options) : PerfTest(options) {}
+
+    /**
+     * @brief Define the test
+     *
+     */
+    void Run(Azure::Core::Context const&) override { auto t = m_client->GetSecret(m_secretName); }
+
+    /**
+     * @brief Define the test options for the test.
+     *
+     * @return The list of test options.
+     */
+    std::vector<Azure::Perf::TestOption> GetTestOptions() override
+    {
+      return {
+          {"vaultUrl", {"--vaultUrl"}, "The Key Vault Account.", 1, false},
+          {"TenantId", {"--tenantId"}, "The tenant Id for the authentication.", 1, false},
+          {"ClientId", {"--clientId"}, "The client Id for the authentication.", 1, false},
+          {"Secret", {"--secret"}, "The secret for authentication.", 1, false, true}};
+    }
+
+    /**
+     * @brief Get the static Test Metadata for the test.
+     *
+     * @return Azure::Perf::TestMetadata describing the test.
+     */
+    static Azure::Perf::TestMetadata GetTestMetadata()
+    {
+      return {"GetSecret", "Get a secret", [](Azure::Perf::TestOptions options) {
+                return std::make_unique<Azure::Security::KeyVault::Secrets::Test::GetSecret>(
+                    options);
+              }};
+    }
+  };
+}}}}} // namespace Azure::Security::KeyVault::Secrets::Test
diff --git a/...ure-security-keyvault-secrets/test/perf/src/azure_security_keyvault_secrets_perf_test.cpp b/...ure-security-keyvault-secrets/test/perf/src/azure_security_keyvault_secrets_perf_test.cpp
@@ -0,0 +1,18 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+#include <azure/perf.hpp>
+
+#include "azure/keyvault/secrets/test/get_secret_test.hpp"
+
+int main(int argc, char** argv)
+{
+
+  // Create the test list
+  std::vector<Azure::Perf::TestMetadata> tests{
+      Azure::Security::KeyVault::Secrets::Test::GetSecret::GetTestMetadata()};
+
+  Azure::Perf::Program::Run(Azure::Core::Context::ApplicationContext, tests, argc, argv);
+
+  return 0;
+}
diff --git a/sdk/keyvault/cleanup/CMakeLists.txt b/sdk/keyvault/cleanup/CMakeLists.txt
@@ -0,0 +1,16 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# SPDX-License-Identifier: MIT
+
+cmake_minimum_required (VERSION 3.13)
+
+project (cleanup-keyvault LANGUAGES CXX)
+set(CMAKE_CXX_STANDARD 14)
+set(CMAKE_CXX_STANDARD_REQUIRED True)
+
+add_executable (
+  cleanup-keyvault
+  cleanup.cpp
+)
+create_per_service_target_build_for_sample(keyvault cleanup-keyvault)
+
+target_link_libraries(cleanup-keyvault PRIVATE azure-security-keyvault-secrets azure-security-keyvault-keys azure-security-keyvault-certificates azure-identity get-env-helper)