Incorporate monitor commands from other repo

.github/workflows/monitor.yaml

@@ -0,0 +1,78 @@
+name: Run every minute
+
+on:
+
+    push:
+        branches:
+            - main
+
+    # Uncomment the following line to run the workflow every 5 minutes
+    # schedule:
+    #     - cron: '5 * * * *'
+
+    workflow_dispatch:
+
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    env:
+        KOSLI_API_TOKEN: ${{ secrets.KOSLI_API_TOKEN_MEEKROSOFT }}
+        KOSLI_ORG: meekrosoft
+
+    defaults:
+      run:
+        working-directory: ./app
+
+    steps:
+
+    - uses: actions/checkout@v4
+
+    - id: 'auth'
+      uses: 'google-github-actions/auth@v2'
+      with:
+        credentials_json: '${{ secrets.gcp_credentials }}'
+
+    - id: 'get-credentials'
+      uses: 'google-github-actions/get-gke-credentials@v2'
+      with:
+        cluster_name: 'autopilot-cluster-1'
+        location: 'us-central1'
+
+    - id: 'install-kubectl'
+      uses: azure/setup-kubectl@v3
+
+    # The KUBECONFIG env var is automatically exported and picked up by kubectl.
+    - id: 'get-pods'
+      run: 'kubectl get nodes'
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        # Semantic version range syntax or exact version of a Python version
+        python-version: '3.x'
+
+    # You can test your matrix by printing the current Python version
+    - name: Display Python version
+      run: | 
+        python -c "import sys; print(sys.version)"
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+
+    - name: setup kosli
+      uses: kosli-dev/setup-cli-action@v2
+      with:
+        version:
+          2.8.8
+
+    - name: Run a one-line script
+      run: |
+        echo "Hello, world!"
+        kosli version
+        echo "Kosli is installed"
+        ./all.sh
+

kosli-config-monitoring/.gitignore

@@ -0,0 +1,2 @@
+snapdir/
+env/

kosli-config-monitoring/1-download-configmaps.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+set -e
+
+# Usage:
+# ./download-configmaps.sh [-o <snapdir>]  [-c <ConfigMap|crdtype..>] <namespaceA,namespaceB> 
+#
+# Takes a list of namespaces to download configmaps/crds from as an arguemnt
+# writes configmaps to the output directory with the namespace.configmapname as the filename
+# output-dir is the directory to write the configmaps to - default is snapdir if not provided
+
+
+
+output_dir=snapdir
+crd_type=ConfigMap
+
+
+# use getops to parse the arguments 
+while getopts ":o:c:" opt; do
+    case ${opt} in
+        o )
+            output_dir=$OPTARG
+            ;;
+        c )
+            crd_type=$OPTARG
+            ;;
+        \? )
+            echo "Usage: cmd [-o output_dir] [-c crd_type] namespace1,namespace2"
+            ;;
+    esac
+done
+
+echo Output directory: $output_dir
+echo CRD Type: $crd_type
+
+# Shift off the options and optional arguments
+shift $((OPTIND -1))
+
+# Now, "$@" contains the additional arguments
+namespaces="$@"
+# Check if the namespace argument is provided
+if [ $# -ne 1 ]; then
+    echo "Please provide a list of namespaces as an argument."
+    exit 1
+fi
+
+echo "Namespaces: $namespaces"
+
+mkdir -p $output_dir
+
+#for each namespace, download the configmaps
+for namespace in $(echo $namespaces | tr "," "\n")
+do
+    echo "Downloading $crd_type's for namespace: $namespace"
+    if [ -z "$(kubectl get namespace $namespace -o name)" ]; then
+        echo "Namespace '$namespace' does not exist."
+        exit 1
+    fi
+
+    # for each namespace, download the configmaps
+    configmaps=$(kubectl get $crd_type -n $namespace -o go-template='{{range .items}}{{.metadata.name}}{{printf "\n"}}{{end}}')  
+
+    for configmap in $configmaps
+    do
+        configmap_json_filename=$output_dir/$namespace.$configmap.json
+        echo Writing: $configmap_json_filename
+        kubectl get $crd_type -n $namespace -o json $configmap > $configmap_json_filename
+
+        # sort the keys in the yaml file
+        tmp_file=$(mktemp)
+        mv $configmap_json_filename $tmp_file
+        jq -S . $tmp_file > $configmap_json_filename
+    done
+done
+

kosli-config-monitoring/2-filter-configmaps.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -e
+
+# Usage:
+# ./2-snapshot-configmaps.sh 
+#
+# 
+
+# Need to have the env activated to run the python script:
+# source env/bin/activate
+
+rm -rf /tmp/snapdir
+mkdir -p /tmp/snapdir
+python src/filter_configmaps.py /tmp/snapdir

kosli-config-monitoring/3-snapshot-configmaps.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -e
+
+# Usage:
+# ./2-snapshot-configmaps.sh 
+#
+# 
+
+kosli snapshot server kubernetes-configurations \
+  --paths "/tmp/snapdir/*"

kosli-config-monitoring/README.md

@@ -0,0 +1,63 @@
+
+
+
+# Tracking kubernetes configmaps with kosli
+
+
+1. download configmaps
+2. filter out generated fields in configmaps
+3. snapshot 
+
+
+# Prerequisites
+
+```bash
+python3 -m venv env
+source env/bin/activate
+python -m pip install --upgrade pip
+pip install -r requirements.txt
+```
+
+
+# Run all - download, filter and report to kosli
+```bash
+./all.sh
+```
+
+
+## 1. Download configmaps
+
+```bash
+gcloud container clusters get-credentials autopilot-cluster-1 --region us-central1 --project test-kubernetes-environment
+
+# Download Configmaps
+./1-download-configmaps.sh -o snapdir gamestore
+
+# Download CRDs
+./1-download-configmaps.sh -o snapdir -c gamestore
+```
+
+## 2. Filter configmaps
+
+```bash
+source env/bin/activate
+
+mkdir /tmp/snapdir
+python src/filter_configmaps.py /tmp/snapdir
+
+```
+
+
+## 3. Snapshot downloaded configs
+
+```bash
+kosli snapshot server kubernetes-configurations \
+  --paths "/tmp/snapdir/*"
+```
+
+
+# Running the tests
+
+```bash
+pytest
+```

kosli-config-monitoring/all.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -e
+
+./1-download-configmaps.sh -o snapdir gamestore
+./2-filter-configmaps.sh   
+./3-snapshot-configmaps.sh

kosli-config-monitoring/configsync-ignore-list.yaml

@@ -0,0 +1,16 @@
+metadata:
+  annotations:
+    config.k8s.io/owning-inventory: "DON'T CARE"
+    configmanagement.gke.io/managed: "DON'T CARE"
+    configmanagement.gke.io/source-path: "DON'T CARE"
+    configmanagement.gke.io/token: "DON'T CARE"
+    configsync.gke.io/declared-fields: "DON'T CARE"
+    configsync.gke.io/git-context: "DON'T CARE"
+    configsync.gke.io/manager: "DON'T CARE"
+    configsync.gke.io/resource-id: "DON'T CARE"
+  creationTimestamp: "DON'T CARE"
+  labels:
+    app.kubernetes.io/managed-by: "DON'T CARE"
+    configsync.gke.io/declared-version: "DON'T CARE"
+  resourceVersion: "DON'T CARE"
+  uid: "DON'T CARE"

kosli-config-monitoring/requirements.txt

@@ -0,0 +1,2 @@
+pytest==6.2.5
+PyYAML

kosli-config-monitoring/src/__init__.py

@@ -0,0 +1 @@
+from .filter_configmaps import load_configmap, load_configmap, visit_leaf_nodes, remove_items

kosli-config-monitoring/src/filter_configmaps.py

@@ -0,0 +1,104 @@
+import json
+import yaml
+
+
+
+def load_configmap(file_path):
+    """Load a ConfigMap from a file"""
+    with open(file_path, "r") as f:
+        return yaml.safe_load(f)
+
+def load_configmap_json(file_path):
+    """Load a ConfigMap from a file"""
+    with open(file_path, "r") as f:
+        return json.load(f)
+
+def visit_leaf_nodes(configmap, ctx=[]):
+        '''Visit all leaf nodes of a nested dictionary  
+            Return a list containing the path to the leaf node and its value
+        '''
+        leaf_nodes = []
+        for key, value in configmap.items():
+            if isinstance(value, dict):
+                # print("visiting dict", key, value)
+                leaf_nodes.extend(visit_leaf_nodes(value, ctx + [key]))
+            else:
+                # print("visiting leaf", key, value)
+                leaf_nodes.append(tuple(ctx + [key, value]))
+        return leaf_nodes
+
+
+def remove_items(initial_configmap, filter_list):
+    """
+    Remove items from the initial_configmap based on the filter_list
+    """
+    filter_items = visit_leaf_nodes(filter_list)
+
+    # for every leaf node in the filter list, remove the corresponding key from the to_be_filtered
+    for leaf_node in filter_items:
+        # pop the last element from the leaf node
+        leaf_node = leaf_node[:-1]
+        print("removing", leaf_node)
+        # remove the key from the deepcopy
+        if(len(leaf_node) == 1):
+            try:
+                initial_configmap.pop(leaf_node[0], None)
+            except KeyError as k:
+                print("KeyError", k)
+        elif(len(leaf_node) == 2):
+            try:
+                initial_configmap[leaf_node[0]].pop(leaf_node[1], None)
+            except KeyError as k:
+                print("KeyError", k)
+        elif(len(leaf_node) == 3):
+            try:
+                initial_configmap[leaf_node[0]][leaf_node[1]].pop(leaf_node[2], None)
+            except KeyError as k:
+                print("KeyError", k)
+
+        elif(len(leaf_node) == 4):
+            try:
+                initial_configmap[leaf_node[0]][leaf_node[1]][leaf_node[2]].pop(leaf_node[3], None)
+            except KeyError as k:
+                print("KeyError", k)
+        elif(len(leaf_node) == 5):
+            try:
+                initial_configmap[leaf_node[0]][leaf_node[1]][leaf_node[2]][leaf_node[3]].pop(leaf_node[4], None)
+            except KeyError as k:
+                print("KeyError", k)
+        else:
+            print("Error: leaf node is too deep")
+            assert False
+
+
+
+# main
+if __name__ == "__main__":
+
+    # get the output directory from the command line argument
+    import sys
+    output_dir = sys.argv[1]
+
+    # create a tmp dir
+    import os
+    import shutil
+    import tempfile
+    tmp_dir = output_dir
+    print("tmp_dir:", tmp_dir)
+
+    # for every file in snapdir, load the configmap and remove the items in filter_list
+    snapdir = "snapdir"
+    for file in os.listdir(snapdir):
+        if file.endswith(".json"):
+            print("processing", file)
+            configmap = load_configmap(os.path.join(snapdir, file))
+            filter_list = load_configmap("configsync-ignore-list.yaml")
+            remove_items(configmap, filter_list)
+            with open(os.path.join(tmp_dir, file), "w") as f:
+
+                json.dump(configmap, f, indent=2)
+                f.write("\n")
+                #yaml.dump(configmap, f)
+
+
+