[COR-37] Integration / functional tests (#23)

.github/workflows/operator.yml

@@ -0,0 +1,113 @@
+name: CoreDB operator workflow
+
+defaults:
+  run:
+    shell: bash
+    working-directory: ./coredb-operator/
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+    - '.github/workflows/operator.yml'
+    - 'coredb-operator/**'
+  # Always run tests on main or release branches
+  push:
+    branches:
+      - main
+      - release**
+
+jobs:
+  lint:
+    name: Run linters
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install minimal nightly with clippy and rustfmt
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly
+          components: rustfmt, clippy
+      - uses: Swatinem/rust-cache@v2
+        with:
+          prefix-key: "coredb-operator-lint"
+          workspaces: |
+            coredb-operator
+      - name: Cargo format
+        run: cargo +nightly fmt --check
+      - name: Clippy
+        run: cargo clippy
+  test:
+    name: Run unit tests
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+      - uses: Swatinem/rust-cache@v2
+        with:
+          prefix-key: "coredb-operator-test"
+          workspaces: |
+            coredb-operator
+      - name: Unit tests
+        run: cargo test
+
+  functional_test:
+    name: Run functional testing
+    runs-on: ubuntu-20.04
+    strategy:
+      # fail-fast means to cancel all jobs if one fails
+      fail-fast: false
+      matrix:
+        # Go here for a list of versions:
+        # https://github.com/kubernetes-sigs/kind/releases
+        node_image:
+          - 'kindest/node:v1.25.3'
+          - 'kindest/node:v1.22.15'
+    steps:
+      - uses: actions/checkout@v2
+      - name: Create k8s Kind Cluster
+        uses: helm/kind-action@v1.5.0
+        with:
+          node_image: ${{ matrix.node_image }}
+      - name: Apply safety check label to namespace
+        run: |
+          kubectl label namespace default safe-to-run-coredb-tests=true
+          kubectl get -o yaml namespaces
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+      - uses: Swatinem/rust-cache@v2
+        with:
+          prefix-key: "coredb-operator-functional-test"
+          workspaces: |
+            coredb-operator
+      - name: Run functional / integration tests
+        run: |
+          set -xe
+          # Install the CRD
+          cargo run --bin crdgen | kubectl apply -f -
+          kubectl get crds
+          # Start the operator in the background
+          cargo run &
+          # Run the tests
+          cargo test -- --ignored --nocapture
+      - name: Debugging information
+        if: always()
+        run: |
+          set +e
+          set -x
+          echo "=========="
+          kubectl get pods --all-namespaces
+          echo "=========="
+          kubectl get -o yaml sts
+          echo "=========="
+          kubectl get -o yaml svc
+          echo "=========="
+          kubectl get -o yaml pods
+          echo "=========="
+          kubectl get -o yaml coredb
+          echo "=========="

.github/workflows/prom_exporter_ext.yml

@@ -7,10 +7,17 @@ defaults:
 
 on:
   pull_request:
-    branches: [ main ]
-    paths: 
+    branches:
+      - main
+    paths:
     - '.github/workflows/postgres_extension.yml'
     - 'extensions/**'
+  # Always run tests on main or release branches
+  push:
+    branches:
+      - main
+      - release**
+
 
 jobs:
   build_and_test:

coredb-operator/Cargo.toml

@@ -47,6 +47,7 @@ thiserror = "1.0.37"
 assert-json-diff = "2.0.2"
 http = "0.2.8"
 hyper = "0.14.23"
+rand = "0.8.5"
 tower-test = "0.4.0"
 
 [dependencies.kube]

coredb-operator/README.md

@@ -25,6 +25,33 @@ rustup component add rustfmt --toolchain nightly
 cargo +nightly fmt
 ```
 
+### Testing
+
+
+#### Unit testing
+
+```
+cargo test
+```
+
+#### Integration testing
+
+- Connect to a cluster that is safe to run the tests against
+- Set your kubecontext to any namespace, and label it to indicate it is safe to run tests against this cluster (do not do against non-test clusters)
+```
+kubectl label namespace default safe-to-run-coredb-tests=true
+```
+
+- Start or install the controller you want to test (see the following sections)
+- Run the integration tests
+```
+cargo test -- --ignored
+```
+- The integration tests assume you already have installed or are running the operator connected to the cluster.
+
+#### Other testing notes
+
+- Include the `--nocapture` flag to show print statements during test runs
 
 ### Cluster
 As an example; install [`kind`](https://kind.sigs.k8s.io/docs/user/quick-start/#installation). Once installed, follow [these instructions](https://kind.sigs.k8s.io/docs/user/local-registry/) to create a kind cluster connected to a local image registry.

coredb-operator/rustfmt.toml

@@ -9,3 +9,4 @@ ignore = [
 comment_width = 110
 max_width = 110
 inline_attribute_width = 80
+edition = "2021"

coredb-operator/tests/README.md

@@ -0,0 +1,5 @@
+# Integration tests
+
+https://doc.rust-lang.org/book/ch11-03-test-organization.html
+
+> In Rust, integration tests are entirely external to your library. They use your library in the same way any other code would, which means they can only call functions that are part of your library’s public API. Their purpose is to test whether many parts of your library work together correctly. Units of code that work correctly on their own could have problems when integrated, so test coverage of the integrated code is important as well. To create integration tests, you first need a tests directory.

coredb-operator/tests/integration_tests.rs

@@ -0,0 +1,119 @@
+// Include the #[ignore] macro on slow tests.
+// That way, 'cargo test' does not run them by default.
+// To run just these tests, use 'cargo test -- --ignored'
+// To run all tests, use 'cargo test -- --include-ignored'
+//
+// https://doc.rust-lang.org/book/ch11-02-running-tests.html
+//
+// These tests assume there is already kubernetes running and you have a context configured.
+// It also assumes that the CRD(s) and operator are already installed for this cluster.
+// In this way, it can be used as a conformance test on a target, separate from installation.
+
+#[cfg(test)]
+mod test {
+
+    use controller::CoreDB;
+    use k8s_openapi::{
+        api::core::v1::{Namespace, Pod},
+        apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition,
+    };
+    use kube::{
+        api::{Patch, PatchParams},
+        runtime::wait::{await_condition, conditions},
+        Api, Client, Config,
+    };
+    use rand::Rng;
+
+    const API_VERSION: &str = "kube.rs/v1";
+
+    #[tokio::test]
+    #[ignore]
+    async fn functional_test_basic_create() {
+        // Initialize the Kubernetes client
+        let client = kube_client().await;
+
+        // Configurations
+        let mut rng = rand::thread_rng();
+        let name = &format!("test-coredb-{}", rng.gen_range(0..100000));
+        let namespace = "default";
+        let kind = "CoreDB";
+        let replicas = 1;
+
+        // Timeout settings while waiting for an event
+        let timeout_seconds = 30;
+
+        // Apply a basic configuration of CoreDB
+        println!("Creating CoreDB resource {}", name);
+        let coredbs: Api<CoreDB> = Api::namespaced(client.clone(), namespace);
+        let coredb_json = serde_json::json!({
+            "apiVersion": API_VERSION,
+            "kind": kind,
+            "metadata": {
+                "name": name
+            },
+            "spec": {
+                "replicas": replicas
+            }
+        });
+        let params = PatchParams::apply("coredb-integration-test");
+        let patch = Patch::Apply(&coredb_json);
+        let _coredb_resource = coredbs.patch(name, &params, &patch).await;
+
+        // Wait for Pod to be created
+
+        let pod_name = format!("{}-0", name);
+        println!("Waiting for pod to be running: {}", pod_name);
+        let pods: Api<Pod> = Api::namespaced(client.clone(), namespace);
+        if let Err(_) = tokio::time::timeout(
+            std::time::Duration::from_secs(timeout_seconds),
+            await_condition(pods, &pod_name, conditions::is_pod_running()),
+        )
+        .await
+        {
+            panic!(
+                "\n\nERROR: Did not find the pod {} to be running after waiting for {} seconds\n\n",
+                pod_name, timeout_seconds
+            )
+        }
+    }
+
+    async fn kube_client() -> kube::Client {
+        // Initialize the Kubernetes client
+        let client_future = Client::try_default();
+        let client = match client_future.await {
+            Ok(wrapped_client) => wrapped_client,
+            Err(_error) => panic!("Please configure your Kubernetes Context"),
+        };
+        // Get the name of the currently selected namespace
+        let selected_namespace = Config::infer().await.unwrap().default_namespace;
+
+        // Next, check that the currently selected namespace is labeled
+        // to allow the running of tests.
+
+        // List the namespaces with the specified labels
+        let namespaces: Api<Namespace> = Api::all(client.clone());
+        let namespace = namespaces.get(&selected_namespace).await.unwrap();
+        let labels = namespace.metadata.labels.unwrap();
+        assert!(
+            labels.contains_key("safe-to-run-coredb-tests"),
+            "expected to find label 'safe-to-run-coredb-tests'"
+        );
+        assert_eq!(
+            labels["safe-to-run-coredb-tests"], "true",
+            "expected to find label 'safe-to-run-coredb-tests' with value 'true'"
+        );
+
+        // Check that the CRD is installed
+        let crds: Api<CustomResourceDefinition> = Api::all(client.clone());
+        if let Err(_) = tokio::time::timeout(
+            std::time::Duration::from_secs(2),
+            await_condition(crds, "coredbs.kube.rs", conditions::is_crd_established()),
+        )
+        .await
+        {
+            panic!("\n\nERROR: Did not find the CRD to be installed.\n\n")
+        }
+
+        return client;
+    }
+}