diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..0f046820f --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +# More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file +# Ignore build and test binaries. +bin/ +testbin/ diff --git a/.github/actions/build-drivercontainer-image/action.yml b/.github/actions/build-drivercontainer-image/action.yml new file mode 100644 index 000000000..a7b84d1af --- /dev/null +++ b/.github/actions/build-drivercontainer-image/action.yml @@ -0,0 +1,25 @@ +name: Build DriverContainer image + +description: This action builds a DriverContainer image for CI + +inputs: + kernel-version: + required: true + description: The kernel version against which the module should be built + +runs: + using: composite + + steps: + - name: Build the kernel module + run: make KERNEL_SRC_DIR="/usr/src/linux-headers-${{ inputs.kernel-version }}" + working-directory: ci/kmm-kmod + shell: bash + + - name: Build the image + run: docker build -t kmm-kmod:local --build-arg=KERNEL_VERSION=${{ inputs.kernel-version }} ci/kmm-kmod + shell: bash + + - name: Export the image + run: docker save -o kmm-kmod_local.tar kmm-kmod:local + shell: bash diff --git a/.github/actions/collect-troubleshooting/action.yml b/.github/actions/collect-troubleshooting/action.yml new file mode 100644 index 000000000..040b426fd --- /dev/null +++ b/.github/actions/collect-troubleshooting/action.yml @@ -0,0 +1,35 @@ +name: Collect troubleshooting data + +description: This action fetches some troubleshooting data for KMMO + +runs: + using: composite + + steps: + - name: Get all resources in the kmm-operator-system namespace + run: kubectl get all -n kmm-operator-system + shell: bash + + - name: Get all resources in the default namespace + run: kubectl get all + shell: bash + + - name: Describe nodes + run: kubectl describe node + shell: bash + + - name: Describe DaemonSets + run: kubectl describe daemonset -l kmm.node.kubernetes.io/module.name + shell: bash + + - name: Describe Pods + run: kubectl describe pod -l kmm.node.kubernetes.io/module.name + shell: bash + + - name: Collect dmesg + run: sudo dmesg + shell: bash + + - name: Get all operator logs + run: kubectl logs deployment.apps/kmm-operator-controller-manager -n kmm-operator-system + shell: bash diff --git a/.github/actions/create-minikube-cluster/action.yml b/.github/actions/create-minikube-cluster/action.yml new file mode 100644 index 000000000..6dcbbc2a0 --- /dev/null +++ b/.github/actions/create-minikube-cluster/action.yml @@ -0,0 +1,32 @@ +name: Create minikube cluster + +description: This action creates a minikube cluster + +inputs: + start-args: + required: false + description: Arguments passed to `minikube start` + +runs: + using: composite + + steps: + - name: Download and install minikube + id: install-minikube + run: | + wget https://storage.googleapis.com/minikube/releases/latest/minikube_latest_amd64.deb + sudo dpkg -i ./minikube_latest_amd64.deb + echo "::set-output name=version::$(minikube version --short)" + shell: bash + + - name: Cache Minikube artifacts + uses: actions/cache@v3 + with: + path: ~/.minikube/cache + key: ${{ runner.os }}-minikube-${{ steps.install-minikube.outputs.version }} + + - name: Start minikube and wait until CoreDNS is available + run: | + minikube start --driver=docker ${{ inputs.start-args }} + kubectl wait --for=condition=available deployment coredns -n kube-system + shell: bash diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..8913f4e7f --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,21 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: gomod + directory: / + schedule: + interval: daily + + - package-ecosystem: docker + directory: / + schedule: + interval: daily + + - package-ecosystem: github-actions + directory: / + schedule: + interval: daily diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 000000000..ae2cc2c28 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,70 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ main ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ main ] + schedule: + - cron: '17 4 * * 6' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [go] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Learn more about CodeQL language support at https://git.io/codeql-language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: ./path/to/local/query, your-org/your-repo/queries@main + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml new file mode 100644 index 000000000..f2374ec9e --- /dev/null +++ b/.github/workflows/docs.yaml @@ -0,0 +1,37 @@ +name: Docs + +on: [pull_request] + +jobs: + build-compare: + name: Build and compare docs + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + cache: pip + python-version: 3.x + + - name: Install mkdocs + run: pip install -r docs/requirements.txt + + - name: Install Java for PlantUML + run: | + sudo apt update + sudo apt install -y default-jre + + - name: Build the docs site + run: make docs + + - name: Check if the output directory changed + run: | + git add -N docs/site + + if ! git diff --quiet docs/site/ ':!docs/site/index.html' ':!docs/site/sitemap.xml*'; then + echo 'Changes detected; please run make -C docs.' + git diff docs/site ':!docs/site/index.html' ':!docs/site/sitemap.xml*' + exit 1 + fi diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml new file mode 100644 index 000000000..7bd432f3d --- /dev/null +++ b/.github/workflows/e2e.yaml @@ -0,0 +1,537 @@ +name: e2e + +on: [pull_request] + +env: + GO_VERSION: 1.18 + +jobs: + build-operator-image: + runs-on: ubuntu-latest + + name: Build the KMMO container image + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Build the image + run: docker build -t kmm:local . + + - name: Export the image + run: docker save -o kmm_local.tar kmm:local + + - name: Upload the image + uses: actions/upload-artifact@v3 + with: + name: ci-images + if-no-files-found: error + path: kmm_local.tar + retention-days: 1 + + e2e: + runs-on: ubuntu-latest + + name: Prebuilt kernel module + + needs: [build-operator-image] + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Create the minikube cluster + uses: ./.github/actions/create-minikube-cluster + + - name: Label the node so that it gets the module + run: kubectl label node minikube wants-module=kmm_ci_a + + - name: Download container images + uses: actions/download-artifact@v3 + with: + name: ci-images + + - name: Save the kernel version + run: echo "KERNEL_VERSION=$(uname -r)" >> $GITHUB_ENV + + - name: Build the DriverContainer image + uses: ./.github/actions/build-drivercontainer-image + with: + kernel-version: ${{ env.KERNEL_VERSION }} + + - name: Import images into minikube + run: | + minikube image load kmm_local.tar + minikube image load kmm-kmod_local.tar + + - name: Cache binaries needed by Makefile + id: cache-bin + uses: actions/cache@v3 + with: + path: ./bin + key: ${{ runner.os }}-bin-${{ env.GO_VERSION }}-${{ hashFiles('Makefile') }} + + - uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO_VERSION }} + if: steps.cache-bin.outputs.cache-hit != 'true' + + - name: Deploy KMMO + run: make deploy + env: + KUSTOMIZE_CONFIG_DEFAULT: ci/install-ci + + - name: Wait until the KMMO Deployment is Available + run: kubectl wait --for condition=Available deployments.apps -n kmm-operator-system kmm-operator-controller-manager + timeout-minutes: 1 + + - name: Describe the Deployment and get its YAML if that failed + run: | + kubectl describe deployments.apps -n kmm-operator-system kmm-operator-controller-manager + kubectl get -o yaml deployments.apps -n kmm-operator-system kmm-operator-controller-manager + if: ${{ failure() }} + + - name: Check that the kmm_ci_a module is not loaded on the node + run: | + if minikube ssh -- lsmod | grep kmm_ci_a; then + echo "Unexpected lsmod output - the module should not be loaded" + exit 1 + fi + + - name: Add an kmm-ci Module that contains a valid mapping + run: | + sed -e "s/KVER_CHANGEME/$(uname -r)/g" \ + -e s/NAME_CHANGEME/kmm-ci/ \ + -e s/KMOD_CHANGEME/kmm_ci_a/ \ + ci/module-kmm-ci.template.yaml | tee module-kmm-ci.yaml + + kubectl apply -f module-kmm-ci.yaml + + - name: Check that the module gets loaded on the node + run: | + until minikube ssh -- lsmod | grep kmm_ci_a; do + sleep 3 + done + timeout-minutes: 1 + + - name: Check that the node gets labeled with the module's name + run: | + until kubectl get node minikube -o jsonpath='{.metadata.labels}' | jq -e 'has("kmm.node.kubernetes.io/kmm-ci.ready")'; do + sleep 3 + done + timeout-minutes: 1 + + - name: Check that the daemon-set for device plugin is running + run: | + until kubectl get ds -l 'kmm.node.kubernetes.io/module.name=kmm-ci,kmm.node.kubernetes.io/role=device-plugin' -ojson | jq -e '.items[] | select(.status.numberReady == 1)'; do + sleep 3 + done + timeout-minutes: 1 + + - name: Remove the Module + run: kubectl delete -f module-kmm-ci.yaml + + - name: Check that the module gets unloaded from the node + run: | + until ! minikube ssh -- lsmod | grep kmm_ci_a; do + sleep 3 + done + timeout-minutes: 1 + + - name: Check that the node gets unlabeled with the module's name + run: | + until ! kubectl get node minikube -o jsonpath='{.metadata.labels}' | jq -e 'has("kmm.node.kubernetes.io/kmm-ci.ready")'; do + sleep 3 + done + timeout-minutes: 1 + + - name: Collect troubleshooting data + uses: ./.github/actions/collect-troubleshooting + if: ${{ always() }} + + e2e-two-nodes: + name: Prebuilt kernel module - two nodes + + runs-on: ubuntu-latest + + needs: [build-operator-image] + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Create the minikube cluster + uses: ./.github/actions/create-minikube-cluster + with: + start-args: --nodes 2 + + - name: Download container images + uses: actions/download-artifact@v3 + with: + name: ci-images + + - name: Save the kernel version + run: echo "KERNEL_VERSION=$(uname -r)" >> $GITHUB_ENV + + - name: Build the DriverContainer image + uses: ./.github/actions/build-drivercontainer-image + with: + kernel-version: ${{ env.KERNEL_VERSION }} + + - name: Import images into minikube + run: | + minikube image load kmm_local.tar + minikube image load kmm-kmod_local.tar + + - name: Cache binaries needed by Makefile + id: cache-bin + uses: actions/cache@v3 + with: + path: ./bin + key: ${{ runner.os }}-bin-${{ env.GO_VERSION }}-${{ hashFiles('Makefile') }} + + - uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO_VERSION }} + if: steps.cache-bin.outputs.cache-hit != 'true' + + - name: Deploy KMMO + run: make deploy + env: + KUSTOMIZE_CONFIG_DEFAULT: ci/install-ci + + - name: Wait until the KMMO Deployment is Available + run: kubectl wait --for condition=Available deployments.apps -n kmm-operator-system kmm-operator-controller-manager + timeout-minutes: 1 + + - name: Describe the Deployment / pods and get their YAML if that failed + run: | + kubectl describe deployments.apps -n kmm-operator-system kmm-operator-controller-manager + kubectl get -o yaml deployments.apps -n kmm-operator-system kmm-operator-controller-manager + + kubectl describe pod -n kmm-operator-system + kubectl get -o yaml pod -n kmm-operator-system + if: ${{ failure() }} + + - name: Create one Module for each node + run: | + # Node minikube gets module a + sed -e s/NAME_CHANGEME/kmm-ci-a/g \ + -e s/KMOD_CHANGEME/kmm_ci_a/g \ + -e "s/KVER_CHANGEME/${KERNEL_VERSION}/g" \ + ci/module-kmm-ci.template.yaml | tee module-kmm-ci-a.yaml + + # Node minikube-m02 gets module b + sed -e s/NAME_CHANGEME/kmm-ci-b/g \ + -e s/KMOD_CHANGEME/kmm_ci_b/g \ + -e "s/KVER_CHANGEME/${KERNEL_VERSION}/g" \ + ci/module-kmm-ci.template.yaml | tee module-kmm-ci-b.yaml + + kubectl apply -f module-kmm-ci-a.yaml -f module-kmm-ci-b.yaml + + - name: Label the first node to have it receive module a + run: kubectl label node minikube wants-module=kmm_ci_a + + - name: Check that module a gets loaded on the first node + run: | + until minikube ssh -- lsmod | grep kmm_ci_a; do + sleep 3 + done + timeout-minutes: 1 + + - name: Label the second node to have it receive module b + run: kubectl label node minikube-m02 wants-module=kmm_ci_b + + - name: Check that module b gets loaded on the second node + run: | + until minikube ssh -n minikube-m02 -- lsmod | grep kmm_ci_b; do + sleep 3 + done + timeout-minutes: 1 + + - name: Remove the wants-module label on the second node + run: kubectl label node minikube-m02 wants-module- + + - name: Check that module b gets unloaded from the second node + run: | + until ! minikube ssh -n minikube-m02 -- lsmod | grep kmm_ci_b; do + sleep 3 + done + timeout-minutes: 1 + + - name: Verify that the DaemonSet gets garbage collected + run: | + # Cannot use kubectl wait because it will fail if there is no initial match + until [ $(kubectl get daemonsets.apps -l 'kmm.node.kubernetes.io/module.name=kmm-ci-b, kmm.node.kubernetes.io/role!=device-plugin' -o go-template='{{ len .items }}') -eq 0 ]; do + sleep 3 + done + timeout-minutes: 1 + + - name: Collect troubleshooting data + uses: ./.github/actions/collect-troubleshooting + if: ${{ always() }} + + in-cluster-build: + runs-on: ubuntu-latest + + name: In-cluster build + + needs: [build-operator-image] + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Create the minikube cluster + uses: ./.github/actions/create-minikube-cluster + with: + start-args: --addons registry,registry-aliases + + - name: Expose the registry outside the cluster + run: kubectl apply -f ci/registry-nodeport.yaml + + - name: Install skopeo + run: | + sudo apt -y update + sudo apt -y install skopeo + + - name: Download container images + uses: actions/download-artifact@v3 + with: + name: ci-images + + - name: Save the kernel version + run: echo "KERNEL_VERSION=$(uname -r)" >> $GITHUB_ENV + + - name: Build the DriverContainer image + uses: ./.github/actions/build-drivercontainer-image + with: + kernel-version: ${{ env.KERNEL_VERSION }} + + - name: Import DriverContainer base into the internal-registry + run: | + MINIKUBE_REGISTRY_EXT="$(minikube service registry-nodeport -n kube-system --format '{{.IP}}:{{.Port}}' --url)" + skopeo copy --dest-tls-verify=false docker-archive:kmm-kmod_local.tar docker://${MINIKUBE_REGISTRY_EXT}/kmm-base:local + + - name: Import the KMMO image into minikube + run: minikube image load kmm_local.tar + + - name: Cache binaries needed by Makefile + id: cache-bin + uses: actions/cache@v3 + with: + path: ./bin + key: ${{ runner.os }}-bin-${{ env.GO_VERSION }}-${{ hashFiles('Makefile') }} + + - uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO_VERSION }} + if: steps.cache-bin.outputs.cache-hit != 'true' + + - name: Deploy KMMO + run: make deploy + env: + KUSTOMIZE_CONFIG_DEFAULT: ci/install-ci + + - name: Wait until the KMMO Deployment is Available + run: kubectl wait --for condition=Available deployments.apps -n kmm-operator-system kmm-operator-controller-manager + timeout-minutes: 1 + + - name: Describe the Deployment / pods and get their YAML if that failed + run: | + kubectl describe deployments.apps -n kmm-operator-system kmm-operator-controller-manager + kubectl get -o yaml deployments.apps -n kmm-operator-system kmm-operator-controller-manager + + kubectl describe pod -n kmm-operator-system + kubectl get -o yaml pod -n kmm-operator-system + if: ${{ failure() }} + + - name: Create a build secret + run: kubectl create secret generic build-secret --from-literal=ci-build-secret=super-secret-value + + # The minikube registry-alias addon creates a Job that adds registry.minikube to the CoreDNS configuration. + # https://github.com/kubernetes/minikube/blob/master/deploy/addons/registry-aliases/patch-coredns-job.tmpl + # This job sometimes does not finish before the operator starts looking for the image in the registry, which + # results in failed DNS resolution. + # Add a job that tries to resolve registry.minikube, like the operator. + # Wait up to 6 minutes, which corresponds to our job's maximum lifetime including backoffs. + - name: Wait for the internal registry to be available in CoreDNS + run: | + kubectl apply -f ci/job-wait-minikube-registry-alias.yaml + kubectl wait --for=condition=Complete --timeout -1s job/wait-minikube-registry-alias + timeout-minutes: 6 + + - name: Check the status of components in the kube-system namespace + run: kubectl get all -n kube-system + if: ${{ failure() }} + + - name: Add an kmm-ci Module that contains a valid mapping + run: | + sed -e "s/KVER_CHANGEME/$(uname -r)/g" ci/module-kmm-ci-build.template.yaml | tee module-kmm-ci.yaml + + kubectl apply -f module-kmm-ci.yaml + + - name: Wait for the job to be created + run: | + until kubectl get job -l kmm.node.kubernetes.io/module.name | grep kmm; do + sleep 3 + done + timeout-minutes: 1 + + - name: Wait for Job completion + run: kubectl wait --for condition=complete job -l kmm.node.kubernetes.io/module.name --timeout=-1s + timeout-minutes: 2 + + - name: Collect job logs + run: | + JOB_NAME=$(kubectl get jobs.batch -l kmm.node.kubernetes.io/module.name --template='{{ (index .items 0).metadata.name }}') + kubectl logs jobs.batch/${JOB_NAME} + if: ${{ always() }} + + - name: Check that the module gets loaded on the node + run: | + until minikube ssh -- lsmod | grep kmm_ci_a; do + sleep 3 + done + timeout-minutes: 1 + + - name: Check that the DriverContainer prints the secret's value to the standard output + run: | + POD_NAME=$(kubectl get pod -l kmm.node.kubernetes.io/module.name --template='{{ (index .items 0).metadata.name }}') + + echo "::group::Looking for the build secret" + + until kubectl exec $POD_NAME -- grep super-secret-value /ci-build-secret; do + sleep 3 + done + + echo "::endgroup::" + echo "::group::Looking for the build argument" + + until kubectl exec $POD_NAME -- grep some-build-arg /build-arg; do + sleep 3 + done + + echo "::endgroup::" + echo "::group::Looking for the kernel version" + + until kubectl exec $POD_NAME -- grep $KERNEL_VERSION /kernel-version; do + sleep 3 + done + + echo "::endgroup::" + echo "::group::Looking for the build argument with a default value" + + until kubectl exec $POD_NAME -- grep default-value /default-value; do + sleep 3 + done + + echo "::endgroup::" + timeout-minutes: 1 + + - name: Remove the Module + run: kubectl delete -f module-kmm-ci.yaml + + - name: Check that the module gets unloaded from the node + run: | + until ! minikube ssh -- lsmod | grep kmm_ci_a; do + sleep 3 + done + timeout-minutes: 1 + + - name: Collect troubleshooting data + uses: ./.github/actions/collect-troubleshooting + if: ${{ always() }} + + e2e-crd-variable: + runs-on: ubuntu-latest + + name: Container image CRD variable + + needs: [build-operator-image] + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Create the minikube cluster + uses: ./.github/actions/create-minikube-cluster + + - name: Download container images + uses: actions/download-artifact@v3 + with: + name: ci-images + + - name: Save the kernel version + run: echo "KERNEL_VERSION=$(uname -r)" >> $GITHUB_ENV + + - name: Build the DriverContainer image + uses: ./.github/actions/build-drivercontainer-image + with: + kernel-version: ${{ env.KERNEL_VERSION }} + + - name: Import images into minikube + run: | + minikube image load kmm_local.tar + minikube image load kmm-kmod_local.tar + + - name: Tag the DriverContainer image with the kernel version + run: minikube image tag kmm-kmod:local kmm-kmod:${{ env.KERNEL_VERSION }} + + - name: Cache binaries needed by Makefile + id: cache-bin + uses: actions/cache@v3 + with: + path: ./bin + key: ${{ runner.os }}-bin-${{ env.GO_VERSION }}-${{ hashFiles('Makefile') }} + + - uses: actions/setup-go@v2 + with: + go-version: ${{ env.GO_VERSION }} + if: steps.cache-bin.outputs.cache-hit != 'true' + + - name: Deploy KMMO + run: make deploy + env: + KUSTOMIZE_CONFIG_DEFAULT: ci/install-ci + + - name: Wait until the KMMO Deployment is Available + run: kubectl wait --for condition=Available deployments.apps -n kmm-operator-system kmm-operator-controller-manager + timeout-minutes: 1 + + - name: Describe the Deployment and get its YAML if that failed + run: | + kubectl describe deployments.apps -n kmm-operator-system kmm-operator-controller-manager + kubectl get -o yaml deployments.apps -n kmm-operator-system kmm-operator-controller-manager + if: ${{ failure() }} + + - name: Check that the kmm_ci_a module is not loaded on the node + run: | + if minikube ssh -- lsmod | grep kmm_ci_a; then + echo "Unexpected lsmod output - the module should not be loaded" + exit 1 + fi + + - name: Add an kmm-ci Module that contains a valid mapping + run: kubectl apply -f ci/module-kmm-ci-variable.yaml + + - name: Check that the module gets loaded on the node + run: | + until minikube ssh -- lsmod | grep kmm_ci_a; do + sleep 3 + done + timeout-minutes: 1 + + - name: Remove the Module + run: kubectl delete -f ci/module-kmm-ci-variable.yaml + + - name: Check that the module gets unloaded from the node + run: | + until ! minikube ssh -- lsmod | grep kmm_ci_a; do + sleep 3 + done + timeout-minutes: 1 + + - name: Collect troubleshooting data + uses: ./.github/actions/collect-troubleshooting + if: ${{ always() }} diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml new file mode 100644 index 000000000..26aad3e2f --- /dev/null +++ b/.github/workflows/test-build.yml @@ -0,0 +1,61 @@ +name: Test and build + +on: + pull_request: + branches: [main] + push: + branches: [main] + +jobs: + test-build: + name: Test and build the code + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-go@v2 + with: + go-version: 1.18 + + - name: Generate code + run: make generate + + - name: Check if generating code changed files + run: | + if ! git diff --exit-code; then + echo "Changes detected; diff:" + git diff + fi + + - name: Run tests + run: make unit-test + + - name: Build + run: make + + - name: Generate a coverage file without mocks + run: | + mv cover.out cover-with-mocks.out + grep -Ev '^.+/mock_.+\.go:.+$' cover-with-mocks.out > cover.out + + - name: Upload the coverage to Codecov + uses: codecov/codecov-action@v3 + + golangci-lint: + name: Run golangci-lint + + runs-on: ubuntu-latest + + steps: + - uses: actions/setup-go@v3 + with: + go-version: 1.18 + + - uses: actions/checkout@v3 + + - name: golangci-lint + uses: golangci/golangci-lint-action@v3 + with: + args: --timeout 3m diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..24e3b4384 --- /dev/null +++ b/.gitignore @@ -0,0 +1,30 @@ +# Bundle files are generated +/bundle + +# Operator's binaries +/manager + +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib +/bin +testbin/* + +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Kubernetes Generated files - skip generated files, except for vendored files + +!vendor/**/zz_generated.* + +# editor and IDE paraphernalia +.idea +*.swp +*.swo +*~ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3cbbabb52..2fe58bdc7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,9 +8,26 @@ _As contributors and maintainers of this project, and in the interest of fosteri We have full documentation on how to get started contributing here: - +### Commit messages +Each PR should consist of a single commit with the following format: +``` +Commit title no longer than 80 chars + +Commit message can be as long as you want. It needs to be complete but minimal, +meaning it should contain all the information to understand the change but +exclude redundant information. Please keep the lines in the commit message up +to 80 chars as well. + +[Jira/GH-issue number] +``` + +The reason for requesting to squash your PR into a single **logical** commit is because our merge policy will squash all the commits anyway before merging. When squashed by Github it will list all the commit titles and messages as bullets, therefore, it will be more informative if correctly written to begin with. + +By using the `squash` merge policy on Github we ensure that each PR is treated as an "atomic" piece of code that can be easily monitored and reverted. + +It is recommended to add the tracking Jira/GH-issue if such exist at the end of the commit message. + +### Kubernetes resources - [Contributor License Agreement](https://git.k8s.io/community/CLA.md) Kubernetes projects require that you sign a Contributor License Agreement (CLA) before we can accept your pull requests - [Kubernetes Contributor Guide](https://git.k8s.io/community/contributors/guide) - Main contributor documentation, or you can just jump directly to the [contributing section](https://git.k8s.io/community/contributors/guide#contributing) @@ -20,12 +37,7 @@ If your repo has certain guidelines for contribution, put them here ahead of the - [Mentoring Initiatives](https://git.k8s.io/community/mentoring) - We have a diverse set of mentorship programs available that are always looking for volunteers! - +- [Slack channel](https://kubernetes.slack.com/messages/sig-node-kmm) +- [Mailing list](https://groups.google.com/g/kubernetes-kmm) diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..d54354b5b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,37 @@ +# Build the manager binary +FROM golang:1.18 as builder + +WORKDIR /workspace + +# Copy the Go Modules manifests +COPY go.mod go.mod +COPY go.sum go.sum + +# cache deps before building and copying source so that we don't need to re-download as much +# and so that source changes don't invalidate our downloaded layer +RUN go mod download + +# Copy the go source +COPY main.go main.go +COPY api api +COPY controllers controllers +COPY internal internal + +# Copy Makefile +COPY Makefile Makefile +COPY docs.mk docs.mk + +# Copy the .git directory which is needed to store the build info +COPY .git .git + +# Build +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 make manager + +# Use distroless as minimal base image to package the manager binary +# Refer to https://github.com/GoogleContainerTools/distroless for more details +FROM gcr.io/distroless/static:nonroot +WORKDIR / +COPY --from=builder /workspace/manager . +USER 65532:65532 + +ENTRYPOINT ["/manager"] diff --git a/Dockerfile.kmm-operator-build b/Dockerfile.kmm-operator-build new file mode 100644 index 000000000..dee519218 --- /dev/null +++ b/Dockerfile.kmm-operator-build @@ -0,0 +1,17 @@ +FROM registry.ci.openshift.org/openshift/release:golang-1.18 + +ENV GO111MODULE=on +ENV GOFLAGS="" +ENV CGO_ENABLED=0 +ENV GOOS=linux +ENV GOARCH=amd64 + +RUN yum install -y which +RUN yum install -y podman docker +RUN go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.6.1 +RUN curl -L --retry 5 "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv4.3.0/kustomize_v4.3.0_linux_amd64.tar.gz" | \ + tar -zx -C /usr/bin +RUN go install github.com/golang/mock/mockgen@v1.5.0 +RUN curl -L "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" -o /tmp/kubectl +RUN install -o root -g root -m 0755 /tmp/kubectl /usr/local/bin/kubectl +COPY --from=quay.io/openshift/origin-cli:latest /usr/bin/oc /usr/bin diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..af127b929 --- /dev/null +++ b/Makefile @@ -0,0 +1,257 @@ +# VERSION defines the project version for the bundle. +# Update this value when you upgrade the version of your project. +# To re-generate a bundle for another specific version without changing the standard setup, you can: +# - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) +# - use environment variables to overwrite this value (e.g export VERSION=0.0.2) +VERSION ?= 0.0.1 + +# CHANNELS define the bundle channels used in the bundle. +# Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") +# To re-generate a bundle for other specific channels without changing the standard setup, you can: +# - use the CHANNELS as arg of the bundle target (e.g make bundle CHANNELS=candidate,fast,stable) +# - use environment variables to overwrite this value (e.g export CHANNELS="candidate,fast,stable") +ifneq ($(origin CHANNELS), undefined) +BUNDLE_CHANNELS := --channels=$(CHANNELS) +endif + +# DEFAULT_CHANNEL defines the default channel used in the bundle. +# Add a new line here if you would like to change its default config. (E.g DEFAULT_CHANNEL = "stable") +# To re-generate a bundle for any other default channel without changing the default setup, you can: +# - use the DEFAULT_CHANNEL as arg of the bundle target (e.g make bundle DEFAULT_CHANNEL=stable) +# - use environment variables to overwrite this value (e.g export DEFAULT_CHANNEL="stable") +ifneq ($(origin DEFAULT_CHANNEL), undefined) +BUNDLE_DEFAULT_CHANNEL := --default-channel=$(DEFAULT_CHANNEL) +endif +BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL) + +# IMAGE_TAG_BASE defines the docker.io namespace and part of the image name for remote images. +# This variable is used to construct full image tags for bundle and catalog images. +# +# For example, running 'make bundle-build bundle-push catalog-build catalog-push' will build and push both +# sigs.k8s.io/oot-operator-bundle:$VERSION and sigs.k8s.io/oot-operator-catalog:$VERSION. +IMAGE_TAG_BASE ?= gcr.io/k8s-staging-kmm/kernel-module-management + +# BUNDLE_IMG defines the image:tag used for the bundle. +# You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=/:) +BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:v$(VERSION) + +# BUNDLE_GEN_FLAGS are the flags passed to the operator-sdk generate bundle command +BUNDLE_GEN_FLAGS ?= -q --overwrite --version $(VERSION) $(BUNDLE_METADATA_OPTS) + +# USE_IMAGE_DIGESTS defines if images are resolved via tags or digests +# You can enable this value if you would like to use SHA Based Digests +# To enable set flag to true +USE_IMAGE_DIGESTS ?= false +ifeq ($(USE_IMAGE_DIGESTS), true) + BUNDLE_GEN_FLAGS += --use-image-digests +endif + +# Image URL to use all building/pushing image targets +IMG ?= $(IMAGE_TAG_BASE):main +# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. +ENVTEST_K8S_VERSION = 1.23 + +# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) +ifeq (,$(shell go env GOBIN)) +GOBIN=$(shell go env GOPATH)/bin +else +GOBIN=$(shell go env GOBIN) +endif + +# Setting SHELL to bash allows bash commands to be executed by recipes. +# This is a requirement for 'setup-envtest.sh' in the test target. +# Options are set to exit when a recipe line exits non-zero or a piped command fails. +SHELL = /usr/bin/env bash -o pipefail +.SHELLFLAGS = -ec + +.PHONY: all +all: generate manager manifests + +##@ General + +# The help target prints out all targets with their descriptions organized +# beneath their categories. The categories are represented by '##@' and the +# target descriptions by '##'. The awk commands is responsible for reading the +# entire set of makefiles included in this invocation, looking for lines of the +# file as xyz: ## something, and then pretty-format the target and help. Then, +# if there's a line with ##@ something, that gets pretty-printed as a category. +# More info on the usage of ANSI control characters for terminal formatting: +# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters +# More info on the awk command: +# http://linuxcommand.org/lc3_adv_awk.php + +.PHONY: help +help: ## Display this help. + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + +##@ Development + +.PHONY: manifests +manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. + $(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases + @for f in config/crd/bases/*.yaml; do \ + kubectl annotate --overwrite -f $$f --local=true -o yaml api-approved.kubernetes.io="unapproved, testing-only" > $$f.bk; \ + mv $$f.bk $$f; \ + done + +.PHONY: generate +generate: controller-gen mockgen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. + $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." + go generate ./... + +.PHONY: fmt +fmt: ## Run go fmt against code. + go fmt ./... + +.PHONY: vet +vet: ## Run go vet against code. + go vet ./... + +TEST ?= ./... + +.PHONY: test +unit-test: vet envtest ## Run tests. + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test $(TEST) -coverprofile cover.out + +.PHONY: lint +lint: golangci-lint ## Run golangci-lint against code. + @if [ `gofmt -l . | wc -l` -ne 0 ]; then \ + echo There are some malformed files, please make sure to run \'make fmt\'; \ + exit 1; \ + fi + $(GOLANGCI_LINT) run -v --timeout 5m0s + +##@ Build + +.PHONY: manager +manager: ## Build manager binary. + go build -o $@ + +.PHONY: run +run: manifests generate fmt vet ## Run a controller from your host. + go run ./main.go + +.PHONY: docker-build +docker-build: unit-test ## Build docker image with the manager. + docker build -t ${IMG} . + +.PHONY: docker-push +docker-push: ## Push docker image with the manager. + docker push ${IMG} + +##@ Deployment + +ifndef ignore-not-found + ignore-not-found = false +endif + +.PHONY: install +install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. + $(KUSTOMIZE) build config/crd | kubectl apply -f - + +.PHONY: uninstall +uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + $(KUSTOMIZE) build config/crd | kubectl delete --ignore-not-found=$(ignore-not-found) -f - + +KUSTOMIZE_CONFIG_DEFAULT ?= config/default + +.PHONY: deploy +deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. + cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} + $(KUSTOMIZE) build $(KUSTOMIZE_CONFIG_DEFAULT) | kubectl apply -f - + +.PHONY: undeploy +undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + $(KUSTOMIZE) build $(KUSTOMIZE_CONFIG_DEFAULT) | kubectl delete --ignore-not-found=$(ignore-not-found) -f - + +CONTROLLER_GEN = $(shell pwd)/bin/controller-gen +.PHONY: controller-gen +controller-gen: ## Download controller-gen locally if necessary. + $(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.8.0) + +GOLANGCI_LINT = $(shell pwd)/bin/golangci-lint +.PHONY: golangci-lint +golangci-lint: ## Download golangci-lint locally if necessary. + $(call go-get-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint@v1.46.2) + +.PHONY: mockgen +mockgen: ## Install mockgen locally. + go install github.com/golang/mock/mockgen@v1.6.0 + +KUSTOMIZE = $(shell pwd)/bin/kustomize +.PHONY: kustomize +kustomize: ## Download kustomize locally if necessary. + $(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v4@v4.5.4) + +ENVTEST = $(shell pwd)/bin/setup-envtest +.PHONY: envtest +envtest: ## Download envtest-setup locally if necessary. + $(call go-get-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest@latest) + +# go-get-tool will 'go get' any package $2 and install it to $1. +PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) +define go-get-tool +@[ -f $(1) ] || { \ +set -e ;\ +echo "Downloading $(2)" ;\ +GOBIN=$(PROJECT_DIR)/bin go install $(2) ;\ +} +endef + +.PHONY: bundle +bundle: manifests kustomize ## Generate bundle manifests and metadata, then validate generated files. + operator-sdk generate kustomize manifests -q + cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) + $(KUSTOMIZE) build config/manifests | operator-sdk generate bundle $(BUNDLE_GEN_FLAGS) + operator-sdk bundle validate ./bundle + +.PHONY: bundle-build +bundle-build: ## Build the bundle image. + docker build -f bundle.Dockerfile -t $(BUNDLE_IMG) . + +.PHONY: bundle-push +bundle-push: ## Push the bundle image. + $(MAKE) docker-push IMG=$(BUNDLE_IMG) + +.PHONY: opm +OPM = ./bin/opm +opm: ## Download opm locally if necessary. +ifeq (,$(wildcard $(OPM))) +ifeq (,$(shell which opm 2>/dev/null)) + @{ \ + set -e ;\ + mkdir -p $(dir $(OPM)) ;\ + OS=$(shell go env GOOS) && ARCH=$(shell go env GOARCH) && \ + curl -sSLo $(OPM) https://github.com/operator-framework/operator-registry/releases/download/v1.19.1/$${OS}-$${ARCH}-opm ;\ + chmod +x $(OPM) ;\ + } +else +OPM = $(shell which opm) +endif +endif + +# A comma-separated list of bundle images (e.g. make catalog-build BUNDLE_IMGS=example.com/operator-bundle:v0.1.0,example.com/operator-bundle:v0.2.0). +# These images MUST exist in a registry and be pull-able. +BUNDLE_IMGS ?= $(BUNDLE_IMG) + +# The image tag given to the resulting catalog image (e.g. make catalog-build CATALOG_IMG=example.com/operator-catalog:v0.2.0). +CATALOG_IMG ?= $(IMAGE_TAG_BASE)-catalog:v$(VERSION) + +# Set CATALOG_BASE_IMG to an existing catalog image tag to add $BUNDLE_IMGS to that image. +ifneq ($(origin CATALOG_BASE_IMG), undefined) +FROM_INDEX_OPT := --from-index $(CATALOG_BASE_IMG) +endif + +# Build a catalog image by adding bundle images to an empty catalog using the operator package manager tool, 'opm'. +# This recipe invokes 'opm' in 'semver' bundle add mode. For more information on add modes, see: +# https://github.com/operator-framework/community-operators/blob/7f1438c/docs/packaging-operator.md#updating-your-existing-operator +.PHONY: catalog-build +catalog-build: opm ## Build a catalog image. + $(OPM) index add --container-tool docker --mode semver --tag $(CATALOG_IMG) --bundles $(BUNDLE_IMGS) $(FROM_INDEX_OPT) + +# Push the catalog image. +.PHONY: catalog-push +catalog-push: ## Push a catalog image. + $(MAKE) docker-push IMG=$(CATALOG_IMG) + +include docs.mk diff --git a/PROJECT b/PROJECT new file mode 100644 index 000000000..0e59a6157 --- /dev/null +++ b/PROJECT @@ -0,0 +1,19 @@ +domain: sigs.k8s.io +layout: +- go.kubebuilder.io/v3 +plugins: + manifests.sdk.operatorframework.io/v2: {} + scorecard.sdk.operatorframework.io/v2: {} +projectName: kernel-module-management +repo: github.com/kubernetes-sigs/kernel-module-management +resources: +- api: + crdVersion: v1 + namespaced: true + controller: true + domain: sigs.k8s.io + group: kmm + kind: Module + path: github.com/kubernetes-sigs/kernel-module-management/api/v1beta1 + version: v1beta1 +version: "3" diff --git a/README.md b/README.md index ba5444fdd..efad61ba2 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,15 @@ -# Kubernetes Template Project - -The Kubernetes Template Project is a template for starting new projects in the GitHub organizations owned by Kubernetes. All Kubernetes projects, at minimum, must have the following files: - -- a `README.md` outlining the project goals, sponsoring sig, and community contact information -- an `OWNERS` with the project leads listed as approvers ([docs on `OWNERS` files][owners]) -- a `CONTRIBUTING.md` outlining how to contribute to the project -- an unmodified copy of `code-of-conduct.md` from this repo, which outlines community behavior and the consequences of breaking the code -- a `LICENSE` which must be Apache 2.0 for code projects, or [Creative Commons 4.0] for documentation repositories, without any custom content -- a `SECURITY_CONTACTS` with the contact points for the Product Security Team - to reach out to for triaging and handling of incoming issues. They must agree to abide by the - [Embargo Policy](https://git.k8s.io/security/private-distributors-list.md#embargo-policy) - and will be removed and replaced if they violate that agreement. +# Kernel Module Management + +[![Go Report Card](https://goreportcard.com/badge/github.com/kubernetes-sigs/kernel-module-management)](https://goreportcard.com/report/github.com/kubernetes-sigs/kernel-module-management) +[![Go Reference](https://pkg.go.dev/badge/github.com/kubernetes-sigs/kernel-module-management.svg)](https://pkg.go.dev/github.com/kubernetes-sigs/kernel-module-management) + +The Out-Of-Tree Operator manages out of tree kernel modules Kubernetes. + +## Getting started +Install the bleeding edge Kernel Module Management Operator in one command: +```shell +kubectl apply -k https://github.com/kubernetes-sigs/kernel-module-management/config/default +``` ## Community, discussion, contribution, and support diff --git a/api/v1beta1/groupversion_info.go b/api/v1beta1/groupversion_info.go new file mode 100644 index 000000000..892c9e17f --- /dev/null +++ b/api/v1beta1/groupversion_info.go @@ -0,0 +1,36 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1beta1 contains API Schema definitions for the kmm v1beta1 API group +//+kubebuilder:object:generate=true +//+groupName=kmm.sigs.k8s.io +package v1beta1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects + GroupVersion = schema.GroupVersion{Group: "kmm.sigs.k8s.io", Version: "v1beta1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/api/v1beta1/module_types.go b/api/v1beta1/module_types.go new file mode 100644 index 000000000..4dbd8ebb0 --- /dev/null +++ b/api/v1beta1/module_types.go @@ -0,0 +1,294 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// BuildArg represents a build argument used when building a container image. +type BuildArg struct { + Name string `json:"name"` + Value string `json:"value"` +} + +type PullOptions struct { + + // +optional + // If Insecure is true, images can be pulled from an insecure (plain HTTP) registry. + Insecure bool `json:"insecure,omitempty"` + + // +optional + // If InsecureSkipTLSVerify, the operator will accept any certificate provided by the registry. + InsecureSkipTLSVerify bool `json:"insecureSkipTLSVerify,omitempty"` +} + +type PushOptions struct { + + // +optional + // If Insecure is true, built images can be pushed to an insecure (plain HTTP) registry. + Insecure bool `json:"insecure,omitempty"` + + // +optional + // If InsecureSkipTLSVerify, the operator will accept any certificate provided by the registry. + InsecureSkipTLSVerify bool `json:"insecureSkipTLSVerify,omitempty"` +} + +type Build struct { + // +optional + // BuildArgs is an array of build variables that are provided to the image building backend. + BuildArgs []BuildArg `json:"buildArgs"` + + Dockerfile string `json:"dockerfile"` + + // +optional + // Pull contains settings determining how to check if the DriverContainer image already exists. + Pull PullOptions `json:"pull"` + + // +optional + // Push contains settings determining how to push a built DriverContainer image. + Push PushOptions `json:"push"` + + // +optional + // Secrets is an optional list of secrets to be made available to the build system. + // Those secrets should be used for private resources such as a private Github repo. + // For container registries auth use module.spec.imagePullSecret instead. + Secrets []v1.LocalObjectReference `json:"secrets"` +} + +// KernelMapping pairs kernel versions with a DriverContainer image. +// Kernel versions can be matched literally or using a regular expression. +type KernelMapping struct { + + // +optional + // Build enables in-cluster builds for this mapping and allows overriding the Module's build settings. + Build *Build `json:"build"` + + // ContainerImage is the name of the DriverContainer image that should be used to deploy the module. + ContainerImage string `json:"containerImage"` + + // +optional + // Literal defines a literal target kernel version to be matched exactly against node kernels. + Literal string `json:"literal"` + + // +optional + // Regexp is a regular expression to be match against node kernels. + Regexp string `json:"regexp"` +} + +type ModprobeArgs struct { + // Load is an optional list of arguments to be used when loading the kernel module. + // +kubebuilder:validation:MinItems=1 + Load []string `json:"load,omitempty"` + + // Unload is an optional list of arguments to be used when unloading the kernel module. + // +kubebuilder:validation:MinItems=1 + Unload []string `json:"unload,omitempty"` +} + +type ModprobeSpec struct { + // ModuleName is the name of the Module to be loaded. + ModuleName string `json:"moduleName"` + + // Parameters is an optional list of kernel module parameters to be provided to modprobe. + // They should be in the form of key=value and will be separated by spaces in the modprobe command. + // The resulting loading command will be: `modprobe module_name ${Parameters}`. + Parameters []string `json:"parameters,omitempty"` + + // DirName is the root directory for modules. + // It adds `-d ${DirName}` to the modprobe command-line. + // +kubebuilder:default=/opt + DirName string `json:"dirName,omitempty"` + + // Args is an optional list of arguments to be passed to modprobe before the name of the kernel module. + // The resulting commands will be: `modprobe ${Args} module_name`. + // +optional + Args *ModprobeArgs `json:"args,omitempty"` + + // If RawArgs are specified, they are passed straight to the modprobe binary; all other properties in this + // object are ignored. + // The resulting commands will be: `modprobe ${RawArgs}`. + // +optional + RawArgs *ModprobeArgs `json:"rawArgs,omitempty"` +} + +type ModuleLoaderContainerSpec struct { + // Build contains build instructions. + // +optional + Build *Build `json:"build,omitempty"` + + // ContainerImage is a top-level field + // +optional + ContainerImage string `json:"containerImage,omitempty"` + + // Image pull policy. + // One of Always, Never, IfNotPresent. + // Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + // Cannot be updated. + // More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + // +optional + ImagePullPolicy v1.PullPolicy `json:"imagePullPolicy,omitempty" protobuf:"bytes,14,opt,name=imagePullPolicy,casttype=PullPolicy"` + + // KernelMappings is a list of kernel mappings. + // When a node's labels match Selector, then the KMM Operator will look for the first mapping that matches its + // kernel version, and use the corresponding container image to run the DriverContainer. + // +kubebuilder:validation:MinItems=1 + KernelMappings []KernelMapping `json:"kernelMappings"` + + // Modprobe is a set of properties to customize which module modprobe loads and with which properties. + Modprobe ModprobeSpec `json:"modprobe"` +} + +type ModuleLoaderSpec struct { + // Container holds the properties for the module loader container that runs modprobe. + Container ModuleLoaderContainerSpec `json:"container"` + + // +optional + // ServiceAccountName is the name of the ServiceAccount to use to run this pod. + // More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ServiceAccountName string `json:"serviceAccountName,omitempty"` +} + +type DevicePluginContainerSpec struct { + // Entrypoint array. Not executed within a shell. + // The container image's ENTRYPOINT is used if this is not provided. + // Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + // cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + // to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + // produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + // of whether the variable exists or not. Cannot be updated. + // More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + // +optional + Command []string `json:"command,omitempty" protobuf:"bytes,3,rep,name=command"` + + // Arguments to the entrypoint. + // The container image's CMD is used if this is not provided. + // Variable references $(VAR_NAME) are expanded using the container's environment. If a variable + // cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced + // to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will + // produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless + // of whether the variable exists or not. Cannot be updated. + // More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell + // +optional + Args []string `json:"args,omitempty" protobuf:"bytes,4,rep,name=args"` + + // List of environment variables to set in the container. + // Cannot be updated. + // +optional + // +patchMergeKey=name + // +patchStrategy=merge + Env []v1.EnvVar `json:"env,omitempty" patchStrategy:"merge" patchMergeKey:"name" protobuf:"bytes,7,rep,name=env"` + + // Image is the name of the container image that the device plugin container will run. + Image string `json:"image"` + + // Image pull policy. + // One of Always, Never, IfNotPresent. + // Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + // Cannot be updated. + // More info: https://kubernetes.io/docs/concepts/containers/images#updating-images + // +optional + ImagePullPolicy v1.PullPolicy `json:"imagePullPolicy,omitempty" protobuf:"bytes,14,opt,name=imagePullPolicy,casttype=PullPolicy"` + + // Compute Resources required by this container. + // Cannot be updated. + // More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + // +optional + Resources v1.ResourceRequirements `json:"resources,omitempty" protobuf:"bytes,8,opt,name=resources"` + + // VolumeMounts is a list of volume mounts that are appended to the default ones. + // +optional + VolumeMounts []v1.VolumeMount `json:"volumeMounts,omitempty"` +} + +type DevicePluginSpec struct { + Container DevicePluginContainerSpec `json:"container"` + + // +optional + // ServiceAccountName is the name of the ServiceAccount to use to run this pod. + // More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + ServiceAccountName string `json:"serviceAccountName,omitempty"` + + Volumes []v1.Volume `json:"volumes,omitempty"` +} + +// ModuleSpec describes how the KMM operator should deploy a Module on those nodes that need it. +type ModuleSpec struct { + // DevicePlugin allows overriding some properties of the container that deploys the device plugin on the node. + // Name is ignored and is set automatically by the KMM Operator. + // +optional + DevicePlugin *DevicePluginSpec `json:"devicePlugin"` + + // ModuleLoader allows overriding some properties of the container that loads the kernel module on the node. + // Name and image are ignored and are set automatically by the KMM Operator. + ModuleLoader ModuleLoaderSpec `json:"moduleLoader"` + + // ImageRepoSecret is an optional secret that is used to pull both the module loader and the device plugin, and + // to push the resulting image from the module loader build, if enabled. + // +optional + ImageRepoSecret *v1.LocalObjectReference `json:"imageRepoSecret,omitempty"` + + // Selector describes on which nodes the Module should be loaded and optionally built. + Selector map[string]string `json:"selector"` +} + +// DaemonSetStatus contains the status for a daemonset deployed during +// reconciliation loop +type DaemonSetStatus struct { + // number of nodes that are targeted by the module selector + NodesMatchingSelectorNumber int32 `json:"nodesMatchingSelectorNumber"` + // number of the pods that should be deployed for daemonset + DesiredNumber int32 `json:"desiredNumber"` + // number of the actually deployed and running pods + AvailableNumber int32 `json:"availableNumber"` +} + +// ModuleStatus defines the observed state of Module. +type ModuleStatus struct { + // DevicePlugin contains the status of the Device Plugin daemonset + // if it was deployed during reconciliation + DevicePlugin DaemonSetStatus `json:"devicePlugin,omitempty"` + // ModuleLoader contains the status of the ModuleLoader daemonset + ModuleLoader DaemonSetStatus `json:"moduleLoader"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:resource:scope=Namespaced +//+kubebuilder:subresource:status + +// Module is the Schema for the modules API +type Module struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec ModuleSpec `json:"spec,omitempty"` + Status ModuleStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// ModuleList contains a list of Module +type ModuleList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Module `json:"items"` +} + +func init() { + SchemeBuilder.Register(&Module{}, &ModuleList{}) +} diff --git a/api/v1beta1/preflightvalidation_types.go b/api/v1beta1/preflightvalidation_types.go new file mode 100644 index 000000000..2a9ee792a --- /dev/null +++ b/api/v1beta1/preflightvalidation_types.go @@ -0,0 +1,108 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + VerificationTrue string = "True" + VerificationFalse string = "False" + VerificationStageImage string = "Image" + VerificationStageBuild string = "Build" +) + +// PreflightValidationSpec describes the desired state of the resource, such as the kernel version +// that Module CRs need to be verified against as well as the debug configuration of the logs +// More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status +// +kubebuilder:validation:Required +type PreflightValidationSpec struct { + // KernelImage describes the kernel image that all Modules need to be checked against. + // +kubebuilder:validation:Required + KernelVersion string `json:"kernelVersion"` +} + +type CRStatus struct { + // Status of Module CR verification: true (verified), false (verification failed), + // error (error during verification process), unknown (verification has not started yet) + // +required + // +kubebuilder:validation:Required + // +kubebuilder:validation:Enum=True;False + VerificationStatus string `json:"verificationStatus"` + + // StatusReason contains a string describing the status source. + // +optional + StatusReason string `json:"statusReason,omitempty"` + + // Current stage of the verification process: + // image (image existence verification), build(build process verification) + // +required + // +kubebuilder:validation:Required + // +kubebuilder:validation:Enum=Image;Build + VerificationStage string `json:"verificationStage"` + + // LastTransitionTime is the last time the CR status transitioned from one status to another. + // This should be when the underlying status changed. If that is not known, then using the time when the API field changed is acceptable. + // +required + // +kubebuilder:validation:Required + // +kubebuilder:validation:Type=string + // +kubebuilder:validation:Format=date-time + LastTransitionTime metav1.Time `json:"lastTransitionTime" protobuf:"bytes,4,opt,name=lastTransitionTime"` +} + +// PreflightValidationStatus is the most recently observed status of the PreflightValidation. +// It is populated by the system and is read-only. +// More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status +type PreflightValidationStatus struct { + // CRStatuses contain observations about each SpecialResource's preflight upgradability validation + // +patchMergeKey=type + // +patchStrategy=merge + // +optional + CRStatuses map[string]*CRStatus `json:"crStatuses,omitempty" patchStrategy:"merge" patchMergeKey:"type"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status + +// PreflightValidation initiates a preflight validations for all SpecialResources on the current Kuberentes cluster. +// +kubebuilder:resource:path=preflightvalidations,scope=Cluster +// +kubebuilder:resource:path=preflightvalidations,scope=Cluster,shortName=pv +type PreflightValidation struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + // +kubebuilder:validation:Required + + Spec PreflightValidationSpec `json:"spec,omitempty"` + Status PreflightValidationStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// PreflightValidationList is a list of PreflightValidation objects. +type PreflightValidationList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + + // List of PreflightValidation. More info: + // https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md + Items []PreflightValidation `json:"items"` +} + +func init() { + SchemeBuilder.Register(&PreflightValidation{}, &PreflightValidationList{}) +} diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go new file mode 100644 index 000000000..828154788 --- /dev/null +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -0,0 +1,525 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1beta1 + +import ( + "k8s.io/api/core/v1" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Build) DeepCopyInto(out *Build) { + *out = *in + if in.BuildArgs != nil { + in, out := &in.BuildArgs, &out.BuildArgs + *out = make([]BuildArg, len(*in)) + copy(*out, *in) + } + out.Pull = in.Pull + out.Push = in.Push + if in.Secrets != nil { + in, out := &in.Secrets, &out.Secrets + *out = make([]v1.LocalObjectReference, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Build. +func (in *Build) DeepCopy() *Build { + if in == nil { + return nil + } + out := new(Build) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BuildArg) DeepCopyInto(out *BuildArg) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BuildArg. +func (in *BuildArg) DeepCopy() *BuildArg { + if in == nil { + return nil + } + out := new(BuildArg) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CRStatus) DeepCopyInto(out *CRStatus) { + *out = *in + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CRStatus. +func (in *CRStatus) DeepCopy() *CRStatus { + if in == nil { + return nil + } + out := new(CRStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DaemonSetStatus) DeepCopyInto(out *DaemonSetStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DaemonSetStatus. +func (in *DaemonSetStatus) DeepCopy() *DaemonSetStatus { + if in == nil { + return nil + } + out := new(DaemonSetStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DevicePluginContainerSpec) DeepCopyInto(out *DevicePluginContainerSpec) { + *out = *in + if in.Command != nil { + in, out := &in.Command, &out.Command + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Args != nil { + in, out := &in.Args, &out.Args + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Env != nil { + in, out := &in.Env, &out.Env + *out = make([]v1.EnvVar, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + in.Resources.DeepCopyInto(&out.Resources) + if in.VolumeMounts != nil { + in, out := &in.VolumeMounts, &out.VolumeMounts + *out = make([]v1.VolumeMount, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DevicePluginContainerSpec. +func (in *DevicePluginContainerSpec) DeepCopy() *DevicePluginContainerSpec { + if in == nil { + return nil + } + out := new(DevicePluginContainerSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DevicePluginSpec) DeepCopyInto(out *DevicePluginSpec) { + *out = *in + in.Container.DeepCopyInto(&out.Container) + if in.Volumes != nil { + in, out := &in.Volumes, &out.Volumes + *out = make([]v1.Volume, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DevicePluginSpec. +func (in *DevicePluginSpec) DeepCopy() *DevicePluginSpec { + if in == nil { + return nil + } + out := new(DevicePluginSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KernelMapping) DeepCopyInto(out *KernelMapping) { + *out = *in + if in.Build != nil { + in, out := &in.Build, &out.Build + *out = new(Build) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KernelMapping. +func (in *KernelMapping) DeepCopy() *KernelMapping { + if in == nil { + return nil + } + out := new(KernelMapping) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ModprobeArgs) DeepCopyInto(out *ModprobeArgs) { + *out = *in + if in.Load != nil { + in, out := &in.Load, &out.Load + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Unload != nil { + in, out := &in.Unload, &out.Unload + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModprobeArgs. +func (in *ModprobeArgs) DeepCopy() *ModprobeArgs { + if in == nil { + return nil + } + out := new(ModprobeArgs) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ModprobeSpec) DeepCopyInto(out *ModprobeSpec) { + *out = *in + if in.Parameters != nil { + in, out := &in.Parameters, &out.Parameters + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Args != nil { + in, out := &in.Args, &out.Args + *out = new(ModprobeArgs) + (*in).DeepCopyInto(*out) + } + if in.RawArgs != nil { + in, out := &in.RawArgs, &out.RawArgs + *out = new(ModprobeArgs) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModprobeSpec. +func (in *ModprobeSpec) DeepCopy() *ModprobeSpec { + if in == nil { + return nil + } + out := new(ModprobeSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Module) DeepCopyInto(out *Module) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Module. +func (in *Module) DeepCopy() *Module { + if in == nil { + return nil + } + out := new(Module) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Module) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ModuleList) DeepCopyInto(out *ModuleList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Module, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModuleList. +func (in *ModuleList) DeepCopy() *ModuleList { + if in == nil { + return nil + } + out := new(ModuleList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ModuleList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ModuleLoaderContainerSpec) DeepCopyInto(out *ModuleLoaderContainerSpec) { + *out = *in + if in.Build != nil { + in, out := &in.Build, &out.Build + *out = new(Build) + (*in).DeepCopyInto(*out) + } + if in.KernelMappings != nil { + in, out := &in.KernelMappings, &out.KernelMappings + *out = make([]KernelMapping, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + in.Modprobe.DeepCopyInto(&out.Modprobe) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModuleLoaderContainerSpec. +func (in *ModuleLoaderContainerSpec) DeepCopy() *ModuleLoaderContainerSpec { + if in == nil { + return nil + } + out := new(ModuleLoaderContainerSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ModuleLoaderSpec) DeepCopyInto(out *ModuleLoaderSpec) { + *out = *in + in.Container.DeepCopyInto(&out.Container) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModuleLoaderSpec. +func (in *ModuleLoaderSpec) DeepCopy() *ModuleLoaderSpec { + if in == nil { + return nil + } + out := new(ModuleLoaderSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ModuleSpec) DeepCopyInto(out *ModuleSpec) { + *out = *in + if in.DevicePlugin != nil { + in, out := &in.DevicePlugin, &out.DevicePlugin + *out = new(DevicePluginSpec) + (*in).DeepCopyInto(*out) + } + in.ModuleLoader.DeepCopyInto(&out.ModuleLoader) + if in.ImageRepoSecret != nil { + in, out := &in.ImageRepoSecret, &out.ImageRepoSecret + *out = new(v1.LocalObjectReference) + **out = **in + } + if in.Selector != nil { + in, out := &in.Selector, &out.Selector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModuleSpec. +func (in *ModuleSpec) DeepCopy() *ModuleSpec { + if in == nil { + return nil + } + out := new(ModuleSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ModuleStatus) DeepCopyInto(out *ModuleStatus) { + *out = *in + out.DevicePlugin = in.DevicePlugin + out.ModuleLoader = in.ModuleLoader +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModuleStatus. +func (in *ModuleStatus) DeepCopy() *ModuleStatus { + if in == nil { + return nil + } + out := new(ModuleStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PreflightValidation) DeepCopyInto(out *PreflightValidation) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PreflightValidation. +func (in *PreflightValidation) DeepCopy() *PreflightValidation { + if in == nil { + return nil + } + out := new(PreflightValidation) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *PreflightValidation) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PreflightValidationList) DeepCopyInto(out *PreflightValidationList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]PreflightValidation, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PreflightValidationList. +func (in *PreflightValidationList) DeepCopy() *PreflightValidationList { + if in == nil { + return nil + } + out := new(PreflightValidationList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *PreflightValidationList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PreflightValidationSpec) DeepCopyInto(out *PreflightValidationSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PreflightValidationSpec. +func (in *PreflightValidationSpec) DeepCopy() *PreflightValidationSpec { + if in == nil { + return nil + } + out := new(PreflightValidationSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PreflightValidationStatus) DeepCopyInto(out *PreflightValidationStatus) { + *out = *in + if in.CRStatuses != nil { + in, out := &in.CRStatuses, &out.CRStatuses + *out = make(map[string]*CRStatus, len(*in)) + for key, val := range *in { + var outVal *CRStatus + if val == nil { + (*out)[key] = nil + } else { + in, out := &val, &outVal + *out = new(CRStatus) + (*in).DeepCopyInto(*out) + } + (*out)[key] = outVal + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PreflightValidationStatus. +func (in *PreflightValidationStatus) DeepCopy() *PreflightValidationStatus { + if in == nil { + return nil + } + out := new(PreflightValidationStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PullOptions) DeepCopyInto(out *PullOptions) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PullOptions. +func (in *PullOptions) DeepCopy() *PullOptions { + if in == nil { + return nil + } + out := new(PullOptions) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PushOptions) DeepCopyInto(out *PushOptions) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PushOptions. +func (in *PushOptions) DeepCopy() *PushOptions { + if in == nil { + return nil + } + out := new(PushOptions) + in.DeepCopyInto(out) + return out +} diff --git a/ci/install-ci/kustomization.yaml b/ci/install-ci/kustomization.yaml new file mode 100644 index 000000000..0909cdc7b --- /dev/null +++ b/ci/install-ci/kustomization.yaml @@ -0,0 +1,24 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../config/default + +images: + - name: gcr.io/k8s-staging-kmm/kernel-module-management + newName: kmm + newTag: local + +patchesStrategicMerge: + - | + apiVersion: apps/v1 + kind: Deployment + metadata: + name: controller-manager + namespace: system + spec: + template: + spec: + containers: + - name: manager + imagePullPolicy: Never diff --git a/ci/job-wait-minikube-registry-alias.yaml b/ci/job-wait-minikube-registry-alias.yaml new file mode 100644 index 000000000..848111298 --- /dev/null +++ b/ci/job-wait-minikube-registry-alias.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: wait-minikube-registry-alias +spec: + template: + spec: + containers: + - name: wait-minikube-registry-alias + image: alpine + command: [nslookup, registry.minikube] + restartPolicy: OnFailure diff --git a/ci/kmm-kmod/.gitignore b/ci/kmm-kmod/.gitignore new file mode 100644 index 000000000..9631110a2 --- /dev/null +++ b/ci/kmm-kmod/.gitignore @@ -0,0 +1,8 @@ +*.cmd +*.ko +*.mod +*.mod.c +*.o + +Module.symvers +modules.order diff --git a/ci/kmm-kmod/Dockerfile b/ci/kmm-kmod/Dockerfile new file mode 100644 index 000000000..99dbec46b --- /dev/null +++ b/ci/kmm-kmod/Dockerfile @@ -0,0 +1,11 @@ +FROM alpine + +RUN ["apk", "add", "kmod"] + +ARG KERNEL_VERSION + +RUN mkdir -p /opt/lib/modules/${KERNEL_VERSION}/extra + +COPY kmm_ci_a.ko kmm_ci_b.ko /opt/lib/modules/${KERNEL_VERSION}/extra/ + +RUN ["depmod", "-b", "/opt"] diff --git a/ci/kmm-kmod/Makefile b/ci/kmm-kmod/Makefile new file mode 100644 index 000000000..c8c08bbc4 --- /dev/null +++ b/ci/kmm-kmod/Makefile @@ -0,0 +1,9 @@ +KERNEL_SRC_DIR = + +ccflags-y += -I$(KERNEL_SRC_DIR)/include +obj-m += kmm_ci_a.o +obj-m += kmm_ci_b.o + +all: + make -C $(KERNEL_SRC_DIR) $(MAKE_OPTS) M=$(PWD) modules + diff --git a/ci/kmm-kmod/kmm_ci_a.c b/ci/kmm-kmod/kmm_ci_a.c new file mode 100644 index 000000000..11832d334 --- /dev/null +++ b/ci/kmm-kmod/kmm_ci_a.c @@ -0,0 +1,20 @@ +#include +#include +#include + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Quentin Barrand"); +MODULE_DESCRIPTION("A simple kernel module for KMM CI"); +MODULE_VERSION("0.01"); + +static int __init kmm_ci_init(void) { + printk(KERN_INFO "Hello, World!\n"); + return 0; +} + +static void __exit kmm_ci_exit(void) { + printk(KERN_INFO "Goodbye, World!\n"); +} + +module_init(kmm_ci_init); +module_exit(kmm_ci_exit); diff --git a/ci/kmm-kmod/kmm_ci_b.c b/ci/kmm-kmod/kmm_ci_b.c new file mode 120000 index 000000000..e42efd7c1 --- /dev/null +++ b/ci/kmm-kmod/kmm_ci_b.c @@ -0,0 +1 @@ +kmm_ci_a.c \ No newline at end of file diff --git a/ci/module-kmm-ci-build.template.yaml b/ci/module-kmm-ci-build.template.yaml new file mode 100644 index 000000000..c7e4b89f0 --- /dev/null +++ b/ci/module-kmm-ci-build.template.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: kmm.sigs.k8s.io/v1beta1 +kind: Module +metadata: + name: kmm-ci-build +spec: + moduleLoader: + container: + modprobe: + moduleName: kmm_ci_a + kernelMappings: + - literal: KVER_CHANGEME + containerImage: registry.minikube/kmm-kmod:local + build: + buildArgs: + - name: CI_BUILD_ARG + value: some-build-arg + pull: + insecure: true + push: + insecure: true + secrets: + - name: build-secret + dockerfile: | + FROM registry.minikube/kmm-base:local + + ARG CI_BUILD_ARG + ARG KERNEL_VERSION + ARG WITH_DEFAULT_VALUE=default-value + + RUN cat /run/secrets/build-secret/ci-build-secret > /ci-build-secret + RUN echo $CI_BUILD_ARG > /build-arg + RUN echo $KERNEL_VERSION > /kernel-version + RUN echo $WITH_DEFAULT_VALUE > /default-value + selector: + kubernetes.io/hostname: minikube diff --git a/ci/module-kmm-ci-variable.yaml b/ci/module-kmm-ci-variable.yaml new file mode 100644 index 000000000..0c8a8c791 --- /dev/null +++ b/ci/module-kmm-ci-variable.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: kmm.sigs.k8s.io/v1beta1 +kind: Module +metadata: + name: kmm-ci +spec: + moduleLoader: + container: + modprobe: + moduleName: kmm_ci_a + imagePullPolicy: Never + kernelMappings: + - regexp: '^.+$' + containerImage: kmm-kmod:$KERNEL_FULL_VERSION + selector: + kubernetes.io/hostname: minikube diff --git a/ci/module-kmm-ci.template.yaml b/ci/module-kmm-ci.template.yaml new file mode 100644 index 000000000..3dec65aab --- /dev/null +++ b/ci/module-kmm-ci.template.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: kmm.sigs.k8s.io/v1beta1 +kind: Module +metadata: + name: NAME_CHANGEME +spec: + moduleLoader: + container: + modprobe: + moduleName: KMOD_CHANGEME + imagePullPolicy: Never + kernelMappings: + - literal: KVER_CHANGEME + containerImage: kmm-kmod:local + devicePlugin: + container: + image: squat/generic-device-plugin + args: + - --device + - '{"name": "simple-kmod", "groups": [{"paths": [{"path": "/proc/simple-procfs-kmod"}]}]}' + volumeMounts: + - name: proc + mountPath: /proc + volumes: + - name: proc + hostPath: + path: /proc + selector: + wants-module: KMOD_CHANGEME diff --git a/ci/prow/lint b/ci/prow/lint index f1f641af1..cbb31536c 100755 --- a/ci/prow/lint +++ b/ci/prow/lint @@ -1 +1,5 @@ #!/usr/bin/env bash + +set -euxo pipefail + +make lint diff --git a/ci/prow/unit-tests b/ci/prow/unit-tests index f1f641af1..4040b1557 100755 --- a/ci/prow/unit-tests +++ b/ci/prow/unit-tests @@ -1 +1,5 @@ #!/usr/bin/env bash + +set -euxo pipefail + +make unit-test diff --git a/ci/registry-nodeport.yaml b/ci/registry-nodeport.yaml new file mode 100644 index 000000000..7e8e0c95e --- /dev/null +++ b/ci/registry-nodeport.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: registry-nodeport + namespace: kube-system +spec: + selector: + actual-registry: 'true' + kubernetes.io/minikube-addons: registry + ports: + - port: 5000 + targetPort: 5000 + type: NodePort diff --git a/config/crd/bases/kmm.sigs.k8s.io_modules.yaml b/config/crd/bases/kmm.sigs.k8s.io_modules.yaml new file mode 100644 index 000000000..4354adffd --- /dev/null +++ b/config/crd/bases/kmm.sigs.k8s.io_modules.yaml @@ -0,0 +1,2175 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: unapproved, testing-only + controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null + name: modules.kmm.sigs.k8s.io +spec: + group: kmm.sigs.k8s.io + names: + kind: Module + listKind: ModuleList + plural: modules + singular: module + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: Module is the Schema for the modules API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ModuleSpec describes how the KMM operator should deploy a + Module on those nodes that need it. + properties: + devicePlugin: + description: DevicePlugin allows overriding some properties of the + container that deploys the device plugin on the node. Name is ignored + and is set automatically by the KMM Operator. + properties: + container: + properties: + args: + description: 'Arguments to the entrypoint. The container image''s + CMD is used if this is not provided. Variable references + $(VAR_NAME) are expanded using the container''s environment. + If a variable cannot be resolved, the reference in the input + string will be unchanged. Double $$ are reduced to a single + $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of + whether the variable exists or not. Cannot be updated. More + info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + command: + description: 'Entrypoint array. Not executed within a shell. + The container image''s ENTRYPOINT is used if this is not + provided. Variable references $(VAR_NAME) are expanded using + the container''s environment. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double + $$ are reduced to a single $, which allows for escaping + the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will produce + the string literal "$(VAR_NAME)". Escaped references will + never be expanded, regardless of whether the variable exists + or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + env: + description: List of environment variables to set in the container. + Cannot be updated. + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must + be a C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded + using the previously defined environment variables + in the container and any service environment variables. + If a variable cannot be resolved, the reference in + the input string will be unchanged. Double $$ are + reduced to a single $, which allows for escaping the + $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will produce + the string literal "$(VAR_NAME)". Escaped references + will never be expanded, regardless of whether the + variable exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports + metadata.name, metadata.namespace, `metadata.labels['''']`, + `metadata.annotations['''']`, spec.nodeName, + spec.serviceAccountName, status.hostIP, status.podIP, + status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in + the specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, limits.ephemeral-storage, requests.cpu, + requests.memory and requests.ephemeral-storage) + are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of + the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + image: + description: Image is the name of the container image that + the device plugin container will run. + type: string + imagePullPolicy: + description: 'Image pull policy. One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent + otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' + type: string + resources: + description: 'Compute Resources required by this container. + Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of + compute resources required. If Requests is omitted for + a container, it defaults to Limits if that is explicitly + specified, otherwise to an implementation-defined value. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + volumeMounts: + description: VolumeMounts is a list of volume mounts that + are appended to the default ones. + items: + description: VolumeMount describes a mounting of a Volume + within a container. + properties: + mountPath: + description: Path within the container at which the + volume should be mounted. Must not contain ':'. + type: string + mountPropagation: + description: mountPropagation determines how mounts + are propagated from the host to container and the + other way around. When not set, MountPropagationNone + is used. This field is beta in 1.10. + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: Mounted read-only if true, read-write otherwise + (false or unspecified). Defaults to false. + type: boolean + subPath: + description: Path within the volume from which the container's + volume should be mounted. Defaults to "" (volume's + root). + type: string + subPathExpr: + description: Expanded path within the volume from which + the container's volume should be mounted. Behaves + similarly to SubPath but environment variable references + $(VAR_NAME) are expanded using the container's environment. + Defaults to "" (volume's root). SubPathExpr and SubPath + are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + required: + - image + type: object + serviceAccountName: + description: 'ServiceAccountName is the name of the ServiceAccount + to use to run this pod. More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/' + type: string + volumes: + items: + description: Volume represents a named volume in a pod that + may be accessed by any container in the pod. + properties: + awsElasticBlockStore: + description: 'awsElasticBlockStore represents an AWS Disk + resource that is attached to a kubelet''s host machine + and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + properties: + fsType: + description: 'fsType is the filesystem type of the volume + that you want to mount. Tip: Ensure that the filesystem + type is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + partition: + description: 'partition is the partition in the volume + that you want to mount. If omitted, the default is + to mount by volume name. Examples: For volume /dev/sda1, + you specify the partition as "1". Similarly, the volume + partition for /dev/sda is "0" (or you can leave the + property empty).' + format: int32 + type: integer + readOnly: + description: 'readOnly value true will force the readOnly + setting in VolumeMounts. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: boolean + volumeID: + description: 'volumeID is unique ID of the persistent + disk resource in AWS (Amazon EBS volume). More info: + https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: string + required: + - volumeID + type: object + azureDisk: + description: azureDisk represents an Azure Data Disk mount + on the host and bind mount to the pod. + properties: + cachingMode: + description: 'cachingMode is the Host Caching mode: + None, Read Only, Read Write.' + type: string + diskName: + description: diskName is the Name of the data disk in + the blob storage + type: string + diskURI: + description: diskURI is the URI of data disk in the + blob storage + type: string + fsType: + description: fsType is Filesystem type to mount. Must + be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. + type: string + kind: + description: 'kind expected values are Shared: multiple + blob disks per storage account Dedicated: single + blob disk per storage account Managed: azure managed + data disk (only in managed availability set). defaults + to shared' + type: string + readOnly: + description: readOnly Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: azureFile represents an Azure File Service + mount on the host and bind mount to the pod. + properties: + readOnly: + description: readOnly defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: secretName is the name of secret that + contains Azure Storage Account Name and Key + type: string + shareName: + description: shareName is the azure share Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: cephFS represents a Ceph FS mount on the host + that shares a pod's lifetime + properties: + monitors: + description: 'monitors is Required: Monitors is a collection + of Ceph monitors More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + items: + type: string + type: array + path: + description: 'path is Optional: Used as the mounted + root, rather than the full Ceph tree, default is /' + type: string + readOnly: + description: 'readOnly is Optional: Defaults to false + (read/write). ReadOnly here will force the ReadOnly + setting in VolumeMounts. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: boolean + secretFile: + description: 'secretFile is Optional: SecretFile is + the path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: string + secretRef: + description: 'secretRef is Optional: SecretRef is reference + to the authentication secret for User, default is + empty. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + user: + description: 'user is optional: User is the rados user + name, default is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: string + required: + - monitors + type: object + cinder: + description: 'cinder represents a cinder volume attached + and mounted on kubelets host machine. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + properties: + fsType: + description: 'fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating + system. Examples: "ext4", "xfs", "ntfs". Implicitly + inferred to be "ext4" if unspecified. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: string + readOnly: + description: 'readOnly defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: boolean + secretRef: + description: 'secretRef is optional: points to a secret + object containing parameters used to connect to OpenStack.' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + volumeID: + description: 'volumeID used to identify the volume in + cinder. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: string + required: + - volumeID + type: object + configMap: + description: configMap represents a configMap that should + populate this volume + properties: + defaultMode: + description: 'defaultMode is optional: mode bits used + to set permissions on created files by default. Must + be an octal value between 0000 and 0777 or a decimal + value between 0 and 511. YAML accepts both octal and + decimal values, JSON requires decimal values for mode + bits. Defaults to 0644. Directories within the path + are not affected by this setting. This might be in + conflict with other options that affect the file mode, + like fsGroup, and the result can be other mode bits + set.' + format: int32 + type: integer + items: + description: items if unspecified, each key-value pair + in the Data field of the referenced ConfigMap will + be projected into the volume as a file whose name + is the key and content is the value. If specified, + the listed keys will be projected into the specified + paths, and unlisted keys will not be present. If a + key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. + Paths must be relative and may not contain the '..' + path or start with '..'. + items: + description: Maps a string key to a path within a + volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: 'mode is Optional: mode bits used + to set permissions on this file. Must be an + octal value between 0000 and 0777 or a decimal + value between 0 and 511. YAML accepts both octal + and decimal values, JSON requires decimal values + for mode bits. If not specified, the volume + defaultMode will be used. This might be in conflict + with other options that affect the file mode, + like fsGroup, and the result can be other mode + bits set.' + format: int32 + type: integer + path: + description: path is the relative path of the + file to map the key to. May not be an absolute + path. May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: optional specify whether the ConfigMap + or its keys must be defined + type: boolean + type: object + csi: + description: csi (Container Storage Interface) represents + ephemeral storage that is handled by certain external + CSI drivers (Beta feature). + properties: + driver: + description: driver is the name of the CSI driver that + handles this volume. Consult with your admin for the + correct name as registered in the cluster. + type: string + fsType: + description: fsType to mount. Ex. "ext4", "xfs", "ntfs". + If not provided, the empty value is passed to the + associated CSI driver which will determine the default + filesystem to apply. + type: string + nodePublishSecretRef: + description: nodePublishSecretRef is a reference to + the secret object containing sensitive information + to pass to the CSI driver to complete the CSI NodePublishVolume + and NodeUnpublishVolume calls. This field is optional, + and may be empty if no secret is required. If the + secret object contains more than one secret, all secret + references are passed. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + readOnly: + description: readOnly specifies a read-only configuration + for the volume. Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: volumeAttributes stores driver-specific + properties that are passed to the CSI driver. Consult + your driver's documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: downwardAPI represents downward API about the + pod that should populate this volume + properties: + defaultMode: + description: 'Optional: mode bits to use on created + files by default. Must be a Optional: mode bits used + to set permissions on created files by default. Must + be an octal value between 0000 and 0777 or a decimal + value between 0 and 511. YAML accepts both octal and + decimal values, JSON requires decimal values for mode + bits. Defaults to 0644. Directories within the path + are not affected by this setting. This might be in + conflict with other options that affect the file mode, + like fsGroup, and the result can be other mode bits + set.' + format: int32 + type: integer + items: + description: Items is a list of downward API volume + file + items: + description: DownwardAPIVolumeFile represents information + to create the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects a field of the + pod: only annotations, labels, name and namespace + are supported.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in + the specified API version. + type: string + required: + - fieldPath + type: object + mode: + description: 'Optional: mode bits used to set + permissions on this file, must be an octal value + between 0000 and 0777 or a decimal value between + 0 and 511. YAML accepts both octal and decimal + values, JSON requires decimal values for mode + bits. If not specified, the volume defaultMode + will be used. This might be in conflict with + other options that affect the file mode, like + fsGroup, and the result can be other mode bits + set.' + format: int32 + type: integer + path: + description: 'Required: Path is the relative + path name of the file to be created. Must not + be absolute or contain the ''..'' path. Must + be utf-8 encoded. The first item of the relative + path must not start with ''..''' + type: string + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, requests.cpu and requests.memory) + are currently supported.' + properties: + containerName: + description: 'Container name: required for + volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of + the exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + emptyDir: + description: 'emptyDir represents a temporary directory + that shares a pod''s lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + properties: + medium: + description: 'medium represents what type of storage + medium should back this directory. The default is + "" which means to use the node''s default medium. + Must be an empty string (default) or Memory. More + info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: 'sizeLimit is the total amount of local + storage required for this EmptyDir volume. The size + limit is also applicable for memory medium. The maximum + usage on memory medium EmptyDir would be the minimum + value between the SizeLimit specified here and the + sum of memory limits of all containers in a pod. The + default is nil which means that the limit is undefined. + More info: http://kubernetes.io/docs/user-guide/volumes#emptydir' + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + description: "ephemeral represents a volume that is handled + by a cluster storage driver. The volume's lifecycle is + tied to the pod that defines it - it will be created before + the pod starts, and deleted when the pod is removed. \n + Use this if: a) the volume is only needed while the pod + runs, b) features of normal volumes like restoring from + snapshot or capacity tracking are needed, c) the storage + driver is specified through a storage class, and d) the + storage driver supports dynamic volume provisioning through + a PersistentVolumeClaim (see EphemeralVolumeSource for + more information on the connection between this volume + type and PersistentVolumeClaim). \n Use PersistentVolumeClaim + or one of the vendor-specific APIs for volumes that persist + for longer than the lifecycle of an individual pod. \n + Use CSI for light-weight local ephemeral volumes if the + CSI driver is meant to be used that way - see the documentation + of the driver for more information. \n A pod can use both + types of ephemeral volumes and persistent volumes at the + same time." + properties: + volumeClaimTemplate: + description: "Will be used to create a stand-alone PVC + to provision the volume. The pod in which this EphemeralVolumeSource + is embedded will be the owner of the PVC, i.e. the + PVC will be deleted together with the pod. The name + of the PVC will be `-` where + `` is the name from the `PodSpec.Volumes` + array entry. Pod validation will reject the pod if + the concatenated name is not valid for a PVC (for + example, too long). \n An existing PVC with that name + that is not owned by the pod will *not* be used for + the pod to avoid using an unrelated volume by mistake. + Starting the pod is then blocked until the unrelated + PVC is removed. If such a pre-created PVC is meant + to be used by the pod, the PVC has to updated with + an owner reference to the pod once the pod exists. + Normally this should not be necessary, but it may + be useful when manually reconstructing a broken cluster. + \n This field is read-only and no changes will be + made by Kubernetes to the PVC after it has been created. + \n Required, must not be nil." + properties: + metadata: + description: May contain labels and annotations + that will be copied into the PVC when creating + it. No other fields are allowed and will be rejected + during validation. + type: object + spec: + description: The specification for the PersistentVolumeClaim. + The entire content is copied unchanged into the + PVC that gets created from this template. The + same fields as in a PersistentVolumeClaim are + also valid here. + properties: + accessModes: + description: 'accessModes contains the desired + access modes the volume should have. More + info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#access-modes-1' + items: + type: string + type: array + dataSource: + description: 'dataSource field can be used to + specify either: * An existing VolumeSnapshot + object (snapshot.storage.k8s.io/VolumeSnapshot) + * An existing PVC (PersistentVolumeClaim) + If the provisioner or an external controller + can support the specified data source, it + will create a new volume based on the contents + of the specified data source. If the AnyVolumeDataSource + feature gate is enabled, this field will always + have the same contents as the DataSourceRef + field.' + properties: + apiGroup: + description: APIGroup is the group for the + resource being referenced. If APIGroup + is not specified, the specified Kind must + be in the core API group. For any other + third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + required: + - kind + - name + type: object + dataSourceRef: + description: 'dataSourceRef specifies the object + from which to populate the volume with data, + if a non-empty volume is desired. This may + be any local object from a non-empty API group + (non core object) or a PersistentVolumeClaim + object. When this field is specified, volume + binding will only succeed if the type of the + specified object matches some installed volume + populator or dynamic provisioner. This field + will replace the functionality of the DataSource + field and as such if both fields are non-empty, + they must have the same value. For backwards + compatibility, both fields (DataSource and + DataSourceRef) will be set to the same value + automatically if one of them is empty and + the other is non-empty. There are two important + differences between DataSource and DataSourceRef: + * While DataSource only allows two specific + types of objects, DataSourceRef allows any + non-core object, as well as PersistentVolumeClaim + objects. * While DataSource ignores disallowed + values (dropping them), DataSourceRef preserves + all values, and generates an error if a disallowed + value is specified. (Beta) Using this field + requires the AnyVolumeDataSource feature gate + to be enabled.' + properties: + apiGroup: + description: APIGroup is the group for the + resource being referenced. If APIGroup + is not specified, the specified Kind must + be in the core API group. For any other + third-party types, APIGroup is required. + type: string + kind: + description: Kind is the type of resource + being referenced + type: string + name: + description: Name is the name of resource + being referenced + type: string + required: + - kind + - name + type: object + resources: + description: 'resources represents the minimum + resources the volume should have. If RecoverVolumeExpansionFailure + feature is enabled users are allowed to specify + resource requirements that are lower than + previous value but must still be higher than + capacity recorded in the status field of the + claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum + amount of compute resources allowed. More + info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum + amount of compute resources required. + If Requests is omitted for a container, + it defaults to Limits if that is explicitly + specified, otherwise to an implementation-defined + value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + selector: + description: selector is a label query over + volumes to consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: operator represents a + key's relationship to a set of values. + Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of + string values. If the operator is + In or NotIn, the values array must + be non-empty. If the operator is + Exists or DoesNotExist, the values + array must be empty. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + storageClassName: + description: 'storageClassName is the name of + the StorageClass required by the claim. More + info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#class-1' + type: string + volumeMode: + description: volumeMode defines what type of + volume is required by the claim. Value of + Filesystem is implied when not included in + claim spec. + type: string + volumeName: + description: volumeName is the binding reference + to the PersistentVolume backing this claim. + type: string + type: object + required: + - spec + type: object + type: object + fc: + description: fc represents a Fibre Channel resource that + is attached to a kubelet's host machine and then exposed + to the pod. + properties: + fsType: + description: 'fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. TODO: how do we prevent + errors in the filesystem from compromising the machine' + type: string + lun: + description: 'lun is Optional: FC target lun number' + format: int32 + type: integer + readOnly: + description: 'readOnly is Optional: Defaults to false + (read/write). ReadOnly here will force the ReadOnly + setting in VolumeMounts.' + type: boolean + targetWWNs: + description: 'targetWWNs is Optional: FC target worldwide + names (WWNs)' + items: + type: string + type: array + wwids: + description: 'wwids Optional: FC volume world wide identifiers + (wwids) Either wwids or combination of targetWWNs + and lun must be set, but not both simultaneously.' + items: + type: string + type: array + type: object + flexVolume: + description: flexVolume represents a generic volume resource + that is provisioned/attached using an exec based plugin. + properties: + driver: + description: driver is the name of the driver to use + for this volume. + type: string + fsType: + description: fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". The default filesystem + depends on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'options is Optional: this field holds + extra command options if any.' + type: object + readOnly: + description: 'readOnly is Optional: defaults to false + (read/write). ReadOnly here will force the ReadOnly + setting in VolumeMounts.' + type: boolean + secretRef: + description: 'secretRef is Optional: secretRef is reference + to the secret object containing sensitive information + to pass to the plugin scripts. This may be empty if + no secret object is specified. If the secret object + contains more than one secret, all secrets are passed + to the plugin scripts.' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + required: + - driver + type: object + flocker: + description: flocker represents a Flocker volume attached + to a kubelet's host machine. This depends on the Flocker + control service being running + properties: + datasetName: + description: datasetName is Name of the dataset stored + as metadata -> name on the dataset for Flocker should + be considered as deprecated + type: string + datasetUUID: + description: datasetUUID is the UUID of the dataset. + This is unique identifier of a Flocker dataset + type: string + type: object + gcePersistentDisk: + description: 'gcePersistentDisk represents a GCE Disk resource + that is attached to a kubelet''s host machine and then + exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + properties: + fsType: + description: 'fsType is filesystem type of the volume + that you want to mount. Tip: Ensure that the filesystem + type is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + partition: + description: 'partition is the partition in the volume + that you want to mount. If omitted, the default is + to mount by volume name. Examples: For volume /dev/sda1, + you specify the partition as "1". Similarly, the volume + partition for /dev/sda is "0" (or you can leave the + property empty). More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + format: int32 + type: integer + pdName: + description: 'pdName is unique name of the PD resource + in GCE. Used to identify the disk in GCE. More info: + https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: string + readOnly: + description: 'readOnly here will force the ReadOnly + setting in VolumeMounts. Defaults to false. More info: + https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: boolean + required: + - pdName + type: object + gitRepo: + description: 'gitRepo represents a git repository at a particular + revision. DEPRECATED: GitRepo is deprecated. To provision + a container with a git repo, mount an EmptyDir into an + InitContainer that clones the repo using git, then mount + the EmptyDir into the Pod''s container.' + properties: + directory: + description: directory is the target directory name. + Must not contain or start with '..'. If '.' is supplied, + the volume directory will be the git repository. Otherwise, + if specified, the volume will contain the git repository + in the subdirectory with the given name. + type: string + repository: + description: repository is the URL + type: string + revision: + description: revision is the commit hash for the specified + revision. + type: string + required: + - repository + type: object + glusterfs: + description: 'glusterfs represents a Glusterfs mount on + the host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/glusterfs/README.md' + properties: + endpoints: + description: 'endpoints is the endpoint name that details + Glusterfs topology. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: string + path: + description: 'path is the Glusterfs volume path. More + info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: string + readOnly: + description: 'readOnly here will force the Glusterfs + volume to be mounted with read-only permissions. Defaults + to false. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: 'hostPath represents a pre-existing file or + directory on the host machine that is directly exposed + to the container. This is generally used for system agents + or other privileged things that are allowed to see the + host machine. Most containers will NOT need this. More + info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + --- TODO(jonesdl) We need to restrict who can use host + directory mounts and who can/can not mount host directories + as read/write.' + properties: + path: + description: 'path of the directory on the host. If + the path is a symlink, it will follow the link to + the real path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + type: + description: 'type for HostPath Volume Defaults to "" + More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + required: + - path + type: object + iscsi: + description: 'iscsi represents an ISCSI Disk resource that + is attached to a kubelet''s host machine and then exposed + to the pod. More info: https://examples.k8s.io/volumes/iscsi/README.md' + properties: + chapAuthDiscovery: + description: chapAuthDiscovery defines whether support + iSCSI Discovery CHAP authentication + type: boolean + chapAuthSession: + description: chapAuthSession defines whether support + iSCSI Session CHAP authentication + type: boolean + fsType: + description: 'fsType is the filesystem type of the volume + that you want to mount. Tip: Ensure that the filesystem + type is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + initiatorName: + description: initiatorName is the custom iSCSI Initiator + Name. If initiatorName is specified with iscsiInterface + simultaneously, new iSCSI interface : will be created for the connection. + type: string + iqn: + description: iqn is the target iSCSI Qualified Name. + type: string + iscsiInterface: + description: iscsiInterface is the interface Name that + uses an iSCSI transport. Defaults to 'default' (tcp). + type: string + lun: + description: lun represents iSCSI Target Lun number. + format: int32 + type: integer + portals: + description: portals is the iSCSI Target Portal List. + The portal is either an IP or ip_addr:port if the + port is other than default (typically TCP ports 860 + and 3260). + items: + type: string + type: array + readOnly: + description: readOnly here will force the ReadOnly setting + in VolumeMounts. Defaults to false. + type: boolean + secretRef: + description: secretRef is the CHAP Secret for iSCSI + target and initiator authentication + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + targetPortal: + description: targetPortal is iSCSI Target Portal. The + Portal is either an IP or ip_addr:port if the port + is other than default (typically TCP ports 860 and + 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: 'name of the volume. Must be a DNS_LABEL and + unique within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + nfs: + description: 'nfs represents an NFS mount on the host that + shares a pod''s lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + properties: + path: + description: 'path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + readOnly: + description: 'readOnly here will force the NFS export + to be mounted with read-only permissions. Defaults + to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: boolean + server: + description: 'server is the hostname or IP address of + the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: 'persistentVolumeClaimVolumeSource represents + a reference to a PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + claimName: + description: 'claimName is the name of a PersistentVolumeClaim + in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + type: string + readOnly: + description: readOnly Will force the ReadOnly setting + in VolumeMounts. Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: photonPersistentDisk represents a PhotonController + persistent disk attached and mounted on kubelets host + machine + properties: + fsType: + description: fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. + type: string + pdID: + description: pdID is the ID that identifies Photon Controller + persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: portworxVolume represents a portworx volume + attached and mounted on kubelets host machine + properties: + fsType: + description: fSType represents the filesystem type to + mount Must be a filesystem type supported by the host + operating system. Ex. "ext4", "xfs". Implicitly inferred + to be "ext4" if unspecified. + type: string + readOnly: + description: readOnly defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: volumeID uniquely identifies a Portworx + volume + type: string + required: + - volumeID + type: object + projected: + description: projected items for all in one resources secrets, + configmaps, and downward API + properties: + defaultMode: + description: defaultMode are the mode bits used to set + permissions on created files by default. Must be an + octal value between 0000 and 0777 or a decimal value + between 0 and 511. YAML accepts both octal and decimal + values, JSON requires decimal values for mode bits. + Directories within the path are not affected by this + setting. This might be in conflict with other options + that affect the file mode, like fsGroup, and the result + can be other mode bits set. + format: int32 + type: integer + sources: + description: sources is the list of volume projections + items: + description: Projection that may be projected along + with other supported volume types + properties: + configMap: + description: configMap information about the configMap + data to project + properties: + items: + description: items if unspecified, each key-value + pair in the Data field of the referenced + ConfigMap will be projected into the volume + as a file whose name is the key and content + is the value. If specified, the listed keys + will be projected into the specified paths, + and unlisted keys will not be present. If + a key is specified which is not present + in the ConfigMap, the volume setup will + error unless it is marked optional. Paths + must be relative and may not contain the + '..' path or start with '..'. + items: + description: Maps a string key to a path + within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: 'mode is Optional: mode + bits used to set permissions on this + file. Must be an octal value between + 0000 and 0777 or a decimal value between + 0 and 511. YAML accepts both octal + and decimal values, JSON requires + decimal values for mode bits. If not + specified, the volume defaultMode + will be used. This might be in conflict + with other options that affect the + file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + path: + description: path is the relative path + of the file to map the key to. May + not be an absolute path. May not contain + the path element '..'. May not start + with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: optional specify whether the + ConfigMap or its keys must be defined + type: boolean + type: object + downwardAPI: + description: downwardAPI information about the + downwardAPI data to project + properties: + items: + description: Items is a list of DownwardAPIVolume + file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: Selects a field + of the pod: only annotations, labels, + name and namespace are supported.' + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to + select in the specified API version. + type: string + required: + - fieldPath + type: object + mode: + description: 'Optional: mode bits used + to set permissions on this file, must + be an octal value between 0000 and + 0777 or a decimal value between 0 + and 511. YAML accepts both octal and + decimal values, JSON requires decimal + values for mode bits. If not specified, + the volume defaultMode will be used. + This might be in conflict with other + options that affect the file mode, + like fsGroup, and the result can be + other mode bits set.' + format: int32 + type: integer + path: + description: 'Required: Path is the + relative path name of the file to + be created. Must not be absolute or + contain the ''..'' path. Must be utf-8 + encoded. The first item of the relative + path must not start with ''..''' + type: string + resourceFieldRef: + description: 'Selects a resource of + the container: only resources limits + and requests (limits.cpu, limits.memory, + requests.cpu and requests.memory) + are currently supported.' + properties: + containerName: + description: 'Container name: required + for volumes, optional for env + vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + secret: + description: secret information about the secret + data to project + properties: + items: + description: items if unspecified, each key-value + pair in the Data field of the referenced + Secret will be projected into the volume + as a file whose name is the key and content + is the value. If specified, the listed keys + will be projected into the specified paths, + and unlisted keys will not be present. If + a key is specified which is not present + in the Secret, the volume setup will error + unless it is marked optional. Paths must + be relative and may not contain the '..' + path or start with '..'. + items: + description: Maps a string key to a path + within a volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: 'mode is Optional: mode + bits used to set permissions on this + file. Must be an octal value between + 0000 and 0777 or a decimal value between + 0 and 511. YAML accepts both octal + and decimal values, JSON requires + decimal values for mode bits. If not + specified, the volume defaultMode + will be used. This might be in conflict + with other options that affect the + file mode, like fsGroup, and the result + can be other mode bits set.' + format: int32 + type: integer + path: + description: path is the relative path + of the file to map the key to. May + not be an absolute path. May not contain + the path element '..'. May not start + with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: optional field specify whether + the Secret or its key must be defined + type: boolean + type: object + serviceAccountToken: + description: serviceAccountToken is information + about the serviceAccountToken data to project + properties: + audience: + description: audience is the intended audience + of the token. A recipient of a token must + identify itself with an identifier specified + in the audience of the token, and otherwise + should reject the token. The audience defaults + to the identifier of the apiserver. + type: string + expirationSeconds: + description: expirationSeconds is the requested + duration of validity of the service account + token. As the token approaches expiration, + the kubelet volume plugin will proactively + rotate the service account token. The kubelet + will start trying to rotate the token if + the token is older than 80 percent of its + time to live or if the token is older than + 24 hours.Defaults to 1 hour and must be + at least 10 minutes. + format: int64 + type: integer + path: + description: path is the path relative to + the mount point of the file to project the + token into. + type: string + required: + - path + type: object + type: object + type: array + type: object + quobyte: + description: quobyte represents a Quobyte mount on the host + that shares a pod's lifetime + properties: + group: + description: group to map volume access to Default is + no group + type: string + readOnly: + description: readOnly here will force the Quobyte volume + to be mounted with read-only permissions. Defaults + to false. + type: boolean + registry: + description: registry represents a single or multiple + Quobyte Registry services specified as a string as + host:port pair (multiple entries are separated with + commas) which acts as the central registry for volumes + type: string + tenant: + description: tenant owning the given Quobyte volume + in the Backend Used with dynamically provisioned Quobyte + volumes, value is set by the plugin + type: string + user: + description: user to map volume access to Defaults to + serivceaccount user + type: string + volume: + description: volume is a string that references an already + created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: 'rbd represents a Rados Block Device mount + on the host that shares a pod''s lifetime. More info: + https://examples.k8s.io/volumes/rbd/README.md' + properties: + fsType: + description: 'fsType is the filesystem type of the volume + that you want to mount. Tip: Ensure that the filesystem + type is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + TODO: how do we prevent errors in the filesystem from + compromising the machine' + type: string + image: + description: 'image is the rados image name. More info: + https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + keyring: + description: 'keyring is the path to key ring for RBDUser. + Default is /etc/ceph/keyring. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + monitors: + description: 'monitors is a collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + items: + type: string + type: array + pool: + description: 'pool is the rados pool name. Default is + rbd. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + readOnly: + description: 'readOnly here will force the ReadOnly + setting in VolumeMounts. Defaults to false. More info: + https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: boolean + secretRef: + description: 'secretRef is name of the authentication + secret for RBDUser. If provided overrides keyring. + Default is nil. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + user: + description: 'user is the rados user name. Default is + admin. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + required: + - image + - monitors + type: object + scaleIO: + description: scaleIO represents a ScaleIO persistent volume + attached and mounted on Kubernetes nodes. + properties: + fsType: + description: fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Default is "xfs". + type: string + gateway: + description: gateway is the host address of the ScaleIO + API Gateway. + type: string + protectionDomain: + description: protectionDomain is the name of the ScaleIO + Protection Domain for the configured storage. + type: string + readOnly: + description: readOnly Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: secretRef references to the secret for + ScaleIO user and other sensitive information. If this + is not provided, Login operation will fail. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + sslEnabled: + description: sslEnabled Flag enable/disable SSL communication + with Gateway, default false + type: boolean + storageMode: + description: storageMode indicates whether the storage + for a volume should be ThickProvisioned or ThinProvisioned. + Default is ThinProvisioned. + type: string + storagePool: + description: storagePool is the ScaleIO Storage Pool + associated with the protection domain. + type: string + system: + description: system is the name of the storage system + as configured in ScaleIO. + type: string + volumeName: + description: volumeName is the name of a volume already + created in the ScaleIO system that is associated with + this volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: 'secret represents a secret that should populate + this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + properties: + defaultMode: + description: 'defaultMode is Optional: mode bits used + to set permissions on created files by default. Must + be an octal value between 0000 and 0777 or a decimal + value between 0 and 511. YAML accepts both octal and + decimal values, JSON requires decimal values for mode + bits. Defaults to 0644. Directories within the path + are not affected by this setting. This might be in + conflict with other options that affect the file mode, + like fsGroup, and the result can be other mode bits + set.' + format: int32 + type: integer + items: + description: items If unspecified, each key-value pair + in the Data field of the referenced Secret will be + projected into the volume as a file whose name is + the key and content is the value. If specified, the + listed keys will be projected into the specified paths, + and unlisted keys will not be present. If a key is + specified which is not present in the Secret, the + volume setup will error unless it is marked optional. + Paths must be relative and may not contain the '..' + path or start with '..'. + items: + description: Maps a string key to a path within a + volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: 'mode is Optional: mode bits used + to set permissions on this file. Must be an + octal value between 0000 and 0777 or a decimal + value between 0 and 511. YAML accepts both octal + and decimal values, JSON requires decimal values + for mode bits. If not specified, the volume + defaultMode will be used. This might be in conflict + with other options that affect the file mode, + like fsGroup, and the result can be other mode + bits set.' + format: int32 + type: integer + path: + description: path is the relative path of the + file to map the key to. May not be an absolute + path. May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + optional: + description: optional field specify whether the Secret + or its keys must be defined + type: boolean + secretName: + description: 'secretName is the name of the secret in + the pod''s namespace to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + type: string + type: object + storageos: + description: storageOS represents a StorageOS volume attached + and mounted on Kubernetes nodes. + properties: + fsType: + description: fsType is the filesystem type to mount. + Must be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. + type: string + readOnly: + description: readOnly defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: secretRef specifies the secret to use for + obtaining the StorageOS API credentials. If not specified, + default values will be attempted. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + volumeName: + description: volumeName is the human-readable name of + the StorageOS volume. Volume names are only unique + within a namespace. + type: string + volumeNamespace: + description: volumeNamespace specifies the scope of + the volume within StorageOS. If no namespace is specified + then the Pod's namespace will be used. This allows + the Kubernetes name scoping to be mirrored within + StorageOS for tighter integration. Set VolumeName + to any name to override the default behaviour. Set + to "default" if you are not using namespaces within + StorageOS. Namespaces that do not pre-exist within + StorageOS will be created. + type: string + type: object + vsphereVolume: + description: vsphereVolume represents a vSphere volume attached + and mounted on kubelets host machine + properties: + fsType: + description: fsType is filesystem type to mount. Must + be a filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. + type: string + storagePolicyID: + description: storagePolicyID is the storage Policy Based + Management (SPBM) profile ID associated with the StoragePolicyName. + type: string + storagePolicyName: + description: storagePolicyName is the storage Policy + Based Management (SPBM) profile name. + type: string + volumePath: + description: volumePath is the path that identifies + vSphere volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + required: + - container + type: object + imageRepoSecret: + description: ImageRepoSecret is an optional secret that is used to + pull both the module loader and the device plugin, and to push the + resulting image from the module loader build, if enabled. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + moduleLoader: + description: ModuleLoader allows overriding some properties of the + container that loads the kernel module on the node. Name and image + are ignored and are set automatically by the KMM Operator. + properties: + container: + description: Container holds the properties for the module loader + container that runs modprobe. + properties: + build: + description: Build contains build instructions. + properties: + buildArgs: + description: BuildArgs is an array of build variables + that are provided to the image building backend. + items: + description: BuildArg represents a build argument used + when building a container image. + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + dockerfile: + type: string + pull: + description: Pull contains settings determining how to + check if the DriverContainer image already exists. + properties: + insecure: + description: If Insecure is true, images can be pulled + from an insecure (plain HTTP) registry. + type: boolean + insecureSkipTLSVerify: + description: If InsecureSkipTLSVerify, the operator + will accept any certificate provided by the registry. + type: boolean + type: object + push: + description: Push contains settings determining how to + push a built DriverContainer image. + properties: + insecure: + description: If Insecure is true, built images can + be pushed to an insecure (plain HTTP) registry. + type: boolean + insecureSkipTLSVerify: + description: If InsecureSkipTLSVerify, the operator + will accept any certificate provided by the registry. + type: boolean + type: object + secrets: + description: Secrets is an optional list of secrets to + be made available to the build system. Those secrets + should be used for private resources such as a private + Github repo. For container registries auth use module.spec.imagePullSecret + instead. + items: + description: LocalObjectReference contains enough information + to let you locate the referenced object inside the + same namespace. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + type: array + required: + - dockerfile + type: object + containerImage: + description: ContainerImage is a top-level field + type: string + imagePullPolicy: + description: 'Image pull policy. One of Always, Never, IfNotPresent. + Defaults to Always if :latest tag is specified, or IfNotPresent + otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' + type: string + kernelMappings: + description: KernelMappings is a list of kernel mappings. + When a node's labels match Selector, then the KMM Operator + will look for the first mapping that matches its kernel + version, and use the corresponding container image to run + the DriverContainer. + items: + description: KernelMapping pairs kernel versions with a + DriverContainer image. Kernel versions can be matched + literally or using a regular expression. + properties: + build: + description: Build enables in-cluster builds for this + mapping and allows overriding the Module's build settings. + properties: + buildArgs: + description: BuildArgs is an array of build variables + that are provided to the image building backend. + items: + description: BuildArg represents a build argument + used when building a container image. + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + dockerfile: + type: string + pull: + description: Pull contains settings determining + how to check if the DriverContainer image already + exists. + properties: + insecure: + description: If Insecure is true, images can + be pulled from an insecure (plain HTTP) registry. + type: boolean + insecureSkipTLSVerify: + description: If InsecureSkipTLSVerify, the operator + will accept any certificate provided by the + registry. + type: boolean + type: object + push: + description: Push contains settings determining + how to push a built DriverContainer image. + properties: + insecure: + description: If Insecure is true, built images + can be pushed to an insecure (plain HTTP) + registry. + type: boolean + insecureSkipTLSVerify: + description: If InsecureSkipTLSVerify, the operator + will accept any certificate provided by the + registry. + type: boolean + type: object + secrets: + description: Secrets is an optional list of secrets + to be made available to the build system. Those + secrets should be used for private resources such + as a private Github repo. For container registries + auth use module.spec.imagePullSecret instead. + items: + description: LocalObjectReference contains enough + information to let you locate the referenced + object inside the same namespace. + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + type: object + type: array + required: + - dockerfile + type: object + containerImage: + description: ContainerImage is the name of the DriverContainer + image that should be used to deploy the module. + type: string + literal: + description: Literal defines a literal target kernel + version to be matched exactly against node kernels. + type: string + regexp: + description: Regexp is a regular expression to be match + against node kernels. + type: string + required: + - containerImage + type: object + minItems: 1 + type: array + modprobe: + description: Modprobe is a set of properties to customize + which module modprobe loads and with which properties. + properties: + args: + description: 'Args is an optional list of arguments to + be passed to modprobe before the name of the kernel + module. The resulting commands will be: `modprobe ${Args} + module_name`.' + properties: + load: + description: Load is an optional list of arguments + to be used when loading the kernel module. + items: + type: string + minItems: 1 + type: array + unload: + description: Unload is an optional list of arguments + to be used when unloading the kernel module. + items: + type: string + minItems: 1 + type: array + type: object + dirName: + default: /opt + description: DirName is the root directory for modules. + It adds `-d ${DirName}` to the modprobe command-line. + type: string + moduleName: + description: ModuleName is the name of the Module to be + loaded. + type: string + parameters: + description: 'Parameters is an optional list of kernel + module parameters to be provided to modprobe. They should + be in the form of key=value and will be separated by + spaces in the modprobe command. The resulting loading + command will be: `modprobe module_name ${Parameters}`.' + items: + type: string + type: array + rawArgs: + description: 'If RawArgs are specified, they are passed + straight to the modprobe binary; all other properties + in this object are ignored. The resulting commands will + be: `modprobe ${RawArgs}`.' + properties: + load: + description: Load is an optional list of arguments + to be used when loading the kernel module. + items: + type: string + minItems: 1 + type: array + unload: + description: Unload is an optional list of arguments + to be used when unloading the kernel module. + items: + type: string + minItems: 1 + type: array + type: object + required: + - moduleName + type: object + required: + - kernelMappings + - modprobe + type: object + serviceAccountName: + description: 'ServiceAccountName is the name of the ServiceAccount + to use to run this pod. More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/' + type: string + required: + - container + type: object + selector: + additionalProperties: + type: string + description: Selector describes on which nodes the Module should be + loaded and optionally built. + type: object + required: + - moduleLoader + - selector + type: object + status: + description: ModuleStatus defines the observed state of Module. + properties: + devicePlugin: + description: DevicePlugin contains the status of the Device Plugin + daemonset if it was deployed during reconciliation + properties: + availableNumber: + description: number of the actually deployed and running pods + format: int32 + type: integer + desiredNumber: + description: number of the pods that should be deployed for daemonset + format: int32 + type: integer + nodesMatchingSelectorNumber: + description: number of nodes that are targeted by the module selector + format: int32 + type: integer + required: + - availableNumber + - desiredNumber + - nodesMatchingSelectorNumber + type: object + moduleLoader: + description: ModuleLoader contains the status of the ModuleLoader + daemonset + properties: + availableNumber: + description: number of the actually deployed and running pods + format: int32 + type: integer + desiredNumber: + description: number of the pods that should be deployed for daemonset + format: int32 + type: integer + nodesMatchingSelectorNumber: + description: number of nodes that are targeted by the module selector + format: int32 + type: integer + required: + - availableNumber + - desiredNumber + - nodesMatchingSelectorNumber + type: object + required: + - moduleLoader + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/crd/bases/kmm.sigs.k8s.io_preflightvalidations.yaml b/config/crd/bases/kmm.sigs.k8s.io_preflightvalidations.yaml new file mode 100644 index 000000000..f2a6634cb --- /dev/null +++ b/config/crd/bases/kmm.sigs.k8s.io_preflightvalidations.yaml @@ -0,0 +1,103 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: unapproved, testing-only + controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null + name: preflightvalidations.kmm.sigs.k8s.io +spec: + group: kmm.sigs.k8s.io + names: + kind: PreflightValidation + listKind: PreflightValidationList + plural: preflightvalidations + shortNames: + - pv + singular: preflightvalidation + scope: Cluster + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: PreflightValidation initiates a preflight validations for all + SpecialResources on the current Kuberentes cluster. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: 'PreflightValidationSpec describes the desired state of the + resource, such as the kernel version that Module CRs need to be verified + against as well as the debug configuration of the logs More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status' + properties: + kernelVersion: + description: KernelImage describes the kernel image that all Modules + need to be checked against. + type: string + required: + - kernelVersion + type: object + status: + description: 'PreflightValidationStatus is the most recently observed + status of the PreflightValidation. It is populated by the system and + is read-only. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status' + properties: + crStatuses: + additionalProperties: + properties: + lastTransitionTime: + description: LastTransitionTime is the last time the CR status + transitioned from one status to another. This should be when + the underlying status changed. If that is not known, then + using the time when the API field changed is acceptable. + format: date-time + type: string + statusReason: + description: StatusReason contains a string describing the status + source. + type: string + verificationStage: + description: 'Current stage of the verification process: image + (image existence verification), build(build process verification)' + enum: + - Image + - Build + type: string + verificationStatus: + description: 'Status of Module CR verification: true (verified), + false (verification failed), error (error during verification + process), unknown (verification has not started yet)' + enum: + - "True" + - "False" + type: string + required: + - lastTransitionTime + - verificationStage + - verificationStatus + type: object + description: CRStatuses contain observations about each SpecialResource's + preflight upgradability validation + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml new file mode 100644 index 000000000..913673321 --- /dev/null +++ b/config/crd/kustomization.yaml @@ -0,0 +1,22 @@ +# This kustomization.yaml is not intended to be run by itself, +# since it depends on service name and namespace that are out of this kustomize package. +# It should be run by config/default +resources: +- bases/kmm.sigs.k8s.io_modules.yaml +- bases/kmm.sigs.k8s.io_preflightvalidations.yaml +#+kubebuilder:scaffold:crdkustomizeresource + +patchesStrategicMerge: +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix. +# patches here are for enabling the conversion webhook for each CRD +#- patches/webhook_in_modules.yaml +#+kubebuilder:scaffold:crdkustomizewebhookpatch + +# [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix. +# patches here are for enabling the CA injection for each CRD +#- patches/cainjection_in_modules.yaml +#+kubebuilder:scaffold:crdkustomizecainjectionpatch + +# the following config is for teaching kustomize how to do kustomization for CRDs. +configurations: +- kustomizeconfig.yaml diff --git a/config/crd/kustomizeconfig.yaml b/config/crd/kustomizeconfig.yaml new file mode 100644 index 000000000..ec5c150a9 --- /dev/null +++ b/config/crd/kustomizeconfig.yaml @@ -0,0 +1,19 @@ +# This file is for teaching kustomize how to substitute name and namespace reference in CRD +nameReference: +- kind: Service + version: v1 + fieldSpecs: + - kind: CustomResourceDefinition + version: v1 + group: apiextensions.k8s.io + path: spec/conversion/webhook/clientConfig/service/name + +namespace: +- kind: CustomResourceDefinition + version: v1 + group: apiextensions.k8s.io + path: spec/conversion/webhook/clientConfig/service/namespace + create: false + +varReference: +- path: metadata/annotations diff --git a/config/crd/patches/cainjection_in_modules.yaml b/config/crd/patches/cainjection_in_modules.yaml new file mode 100644 index 000000000..816209d7b --- /dev/null +++ b/config/crd/patches/cainjection_in_modules.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: modules.kmm.sigs.k8s.io diff --git a/config/crd/patches/webhook_in_modules.yaml b/config/crd/patches/webhook_in_modules.yaml new file mode 100644 index 000000000..e6c58b955 --- /dev/null +++ b/config/crd/patches/webhook_in_modules.yaml @@ -0,0 +1,16 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: modules.kmm.sigs.k8s.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert + conversionReviewVersions: + - v1 diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml new file mode 100644 index 000000000..2408461f0 --- /dev/null +++ b/config/default/kustomization.yaml @@ -0,0 +1,74 @@ +# Adds namespace to all resources. +namespace: kmm-operator-system + +# Value of this field is prepended to the +# names of all resources, e.g. a deployment named +# "wordpress" becomes "alices-wordpress". +# Note that it should also match with the prefix (text before '-') of the namespace +# field above. +namePrefix: kmm-operator- + +# Labels to add to all resources and selectors. +#commonLabels: +# someName: someValue + +bases: +- ../crd +- ../rbac +- ../manager +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in +# crd/kustomization.yaml +#- ../webhook +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. +#- ../certmanager +# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. +#- ../prometheus + +patchesStrategicMerge: +# Protect the /metrics endpoint by putting it behind auth. +# If you want your controller-manager to expose the /metrics +# endpoint w/o any authn/z, please comment the following line. +- manager_auth_proxy_patch.yaml + +# Mount the controller config file for loading manager configurations +# through a ComponentConfig type +#- manager_config_patch.yaml + +# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in +# crd/kustomization.yaml +#- manager_webhook_patch.yaml + +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. +# Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. +# 'CERTMANAGER' needs to be enabled to use ca injection +#- webhookcainjection_patch.yaml + +# the following config is for teaching kustomize how to do var substitution +vars: +# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. +#- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR +# objref: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert # this name should match the one in certificate.yaml +# fieldref: +# fieldpath: metadata.namespace +#- name: CERTIFICATE_NAME +# objref: +# kind: Certificate +# group: cert-manager.io +# version: v1 +# name: serving-cert # this name should match the one in certificate.yaml +#- name: SERVICE_NAMESPACE # namespace of the service +# objref: +# kind: Service +# version: v1 +# name: webhook-service +# fieldref: +# fieldpath: metadata.namespace +#- name: SERVICE_NAME +# objref: +# kind: Service +# version: v1 +# name: webhook-service diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml new file mode 100644 index 000000000..45be31887 --- /dev/null +++ b/config/default/manager_auth_proxy_patch.yaml @@ -0,0 +1,34 @@ +# This patch inject a sidecar container which is a HTTP proxy for the +# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: kube-rbac-proxy + image: gcr.io/kubebuilder/kube-rbac-proxy:v0.8.0 + args: + - "--secure-listen-address=0.0.0.0:8443" + - "--upstream=http://127.0.0.1:8080/" + - "--logtostderr=true" + - "--v=0" + ports: + - containerPort: 8443 + protocol: TCP + name: https + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 5m + memory: 64Mi + - name: manager + args: + - "--health-probe-bind-address=:8081" + - "--metrics-bind-address=127.0.0.1:8080" + - "--leader-elect" diff --git a/config/default/manager_config_patch.yaml b/config/default/manager_config_patch.yaml new file mode 100644 index 000000000..6c400155c --- /dev/null +++ b/config/default/manager_config_patch.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + args: + - "--config=controller_manager_config.yaml" + volumeMounts: + - name: manager-config + mountPath: /controller_manager_config.yaml + subPath: controller_manager_config.yaml + volumes: + - name: manager-config + configMap: + name: manager-config diff --git a/config/manager/controller_manager_config.yaml b/config/manager/controller_manager_config.yaml new file mode 100644 index 000000000..38047017a --- /dev/null +++ b/config/manager/controller_manager_config.yaml @@ -0,0 +1,11 @@ +apiVersion: controller-runtime.sigs.k8s.io/v1alpha1 +kind: ControllerManagerConfig +health: + healthProbeBindAddress: :8081 +metrics: + bindAddress: 127.0.0.1:8080 +webhook: + port: 9443 +leaderElection: + leaderElect: true + resourceName: c5baf8af.sigs.k8s.io diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml new file mode 100644 index 000000000..cfe78b45e --- /dev/null +++ b/config/manager/kustomization.yaml @@ -0,0 +1,16 @@ +resources: +- manager.yaml + +generatorOptions: + disableNameSuffixHash: true + +configMapGenerator: +- files: + - controller_manager_config.yaml + name: manager-config +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: controller + newName: gcr.io/k8s-staging-kmm/kernel-module-management + newTag: main diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml new file mode 100644 index 000000000..e8846f884 --- /dev/null +++ b/config/manager/manager.yaml @@ -0,0 +1,78 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + control-plane: controller-manager + name: system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system + labels: + control-plane: controller-manager +spec: + selector: + matchLabels: + control-plane: controller-manager + replicas: 1 + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: manager + labels: + control-plane: controller-manager + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + securityContext: + runAsNonRoot: true + containers: + - command: + - /manager + args: + - --leader-elect + image: controller:latest + name: manager + imagePullPolicy: Always + securityContext: + allowPrivilegeEscalation: false + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + # TODO(user): Configure the resources accordingly based on the project requirements. + # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 10m + memory: 64Mi + serviceAccountName: controller-manager + terminationGracePeriodSeconds: 10 + tolerations: + - key: node-role.kubernetes.io/master + operator: Equal + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + operator: Equal + effect: NoSchedule diff --git a/config/manifests/bases/kmm-operator.clusterserviceversion.yaml b/config/manifests/bases/kmm-operator.clusterserviceversion.yaml new file mode 100644 index 000000000..45e2fd95d --- /dev/null +++ b/config/manifests/bases/kmm-operator.clusterserviceversion.yaml @@ -0,0 +1,52 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: ClusterServiceVersion +metadata: + annotations: + alm-examples: '[]' + capabilities: Basic Install + name: kmm-operator.v0.0.0 + namespace: placeholder +spec: + apiservicedefinitions: {} + customresourcedefinitions: + owned: + - description: Module is the Schema for the modules API + displayName: Module + kind: Module + name: modules.kmm.sigs.k8s.io + version: v1alpha1 + description: Kubernetes operator managing out of tree kernel modules + displayName: Kernel Module Management + icon: + - base64data: "iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAYAAACqaXHeAAAE8HpUWHRSYXcgcHJvZmlsZSB0eXBlIGV4aWYAAHjarVdZtuMoDP1nFb0ESyAEy2E8p3bQy+8rTFKZXuWlXtsnFgFZSLoasBv//pruH1xMmlwQTTHHeOAKOWQuGKTjvMp60hHWc11hL+H/3by7LjCmPKg//6a4+S/zdBVwkoKR3AhKbS/U+4W8d+D0IGhv5E0jxqBvQXkL8nwu0BZQTrOOmJPemlDHSfvFknT+nD28LtlXIY//g8J7XTDpmYcnf+Dp/VbA24+dLxjoegZeU2uc8WQftyZwyCs/Xa8MjebYUDwz3aFyHdHrefeIVuDN4h+cHK/05bwjeY3Kcv1t/KQ94vt5uEpPjR68b785e5rLZlhRQoSr4zbqYsoaga9iC9s6OagWITMihhKo3Rl3QlQ3oNaPdlTcjTIx4JoUqFOhSWPRRg0qBh6OFQPmxn5NJq+cuQE38sFumqzAsPsEFNuCPXi+6kJr23w0t3ZL2LkTWJkgjFYQfHi7T1+Y01KB6EhXX0EvZnM21DDk7Ak2IEJzO1WWgy/342W4eiAo5mVLkQzH1lNEFfpdCfwC2oNRQM90Ie1bAFyErQXKkAcCQI28UKRDmZUIjkwAqEB1Rs5UIEAi3KEkB+8jsElsW+MVpcXKwph2mEcxAxLiI3IuWZYBrBAE8aMhIYaKeAkiEkUlSZYSfQxRYowarSgW9RqcikZVTZq1JJ9CkhSTppRyKpmzR9GUHLPmlHMuBXsWSC54u4ChlMrV11DF1Vi1pppraQifFpq02LSlllvp3H1H/eixa0899zJoIJRGGDLi0JFGHmUi1KZ3M0yZcepMM89yRW3D+nR/gBpt1HghZYx6RQ2zqhcRZOVEDDMAxi4QEFeDAAHNhtmRKAQ25AyzIzOyQhhKimHWyRADgmEQy6QLdo5PRA25H+HmNNzhxn+LnDPoPkTuGbdXqHVrQ20hdmahOfXwyD6sj1Q4FWt2T9R9tXBPYdgbRle+J+ktfRKEehjEOmM5Buc1j/F76r7L+E1B0jovPfC40gM6bW7lMkabPjXM+vjCQPc3HkEknhuLnQ9O6i6Dr6jFI8LvtUC0R8T9MsHd2nJLZxq+zzyQSiVeZI8ovaaB/lRiv3/LvRYzJPaa0cRCL4z8UEymcPqW0uZCjUYOaJ5Vhzjs0Ka2HvxvUUidOEYPdm47lXnvPocj3ivEHpETiy+UgVJx4bTGj8xf+uhTaoJK6wbMg76mAlRpHSoI/xlaPNzLBd/7idYG/9kn6nn0doHvVRz5LIpCl5tYAGeWVFtBceq5zYrzaB01UhnQk04WwFiOFdnfTSqJ2B+N+jWPu5lQHGN7m2TqMt5BySOs5dFLn6NTrUOhiVKvHucBwuGMQ69hGeMQ5j9P/oPaQm3FCv2suLn71LuJDoJHPlDL/dQuCnllhrsPI5Wrukqx915yPRXclff7GlFrBkzrZ4x8ZNrO2z9mAJcanzVCD7faywv+TzOUK5JjzJsac4kj+yyjfPrmgOceX8Z3yxsrDz01ujPi+CpYtlx8mrxsd84GuymaoPy30eD+h974XpAMBBFxwJEuI1tHbNF3Ox+ZtVJ624bHm8JWke+Tm+DY43NDlhecpo6Q75j/VCXd8a5DPtPL8eIWisN5/4B2mLHHicPYmLlGq6foz1ZzDo65Cqoh+mS+73BfN8hLC8QJzj7F/wN6H/OooEXFkAAAAYNpQ0NQSUNDIHByb2ZpbGUAAHicfZE9SMNAHMVfP6QiFQcziChkqE4WREUctQpFqBBqhVYdTC79giYNSYqLo+BacPBjserg4qyrg6sgCH6AODo5KbpIif9LCi1iPDjux7t7j7t3QLBRYZoVHgc03TbTyYSYza2KkVdEEIaAEIZlZhlzkpSC7/i6R4Cvd3Ge5X/uz9Gr5i0GBETiWWaYNvEG8fSmbXDeJxZYSVaJz4nHTLog8SPXFY/fOBddDvJMwcyk54kFYrHYwUoHs5KpEU8Rx1RNp/xg1mOV8xZnrVJjrXvyF0bz+soy12kOIYlFLEGCCAU1lFGBjTitOikW0rSf8PEPun6JXAq5ymDkWEAVGmTXD/4Hv7u1CpMTXlI0AXS9OM7HCBDZBZp1x/k+dpzmCRB6Bq70tr/aAGY+Sa+3tdgR0LcNXFy3NWUPuNwBBp4M2ZRdKUQzWCgA72f0TTmg/xboWfN6a+3j9AHIUFepG+DgEBgtUva6z7u7O3v790yrvx8MlnJ+ipn2uAAADf1pVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+Cjx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IlhNUCBDb3JlIDQuNC4wLUV4aXYyIj4KIDxyZGY6UkRGIHhtbG5zOnJkZj0iaHR0cDovL3d3dy53My5vcmcvMTk5OS8wMi8yMi1yZGYtc3ludGF4LW5zIyI+CiAgPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIKICAgIHhtbG5zOnhtcE1NPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvbW0vIgogICAgeG1sbnM6c3RFdnQ9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9zVHlwZS9SZXNvdXJjZUV2ZW50IyIKICAgIHhtbG5zOmRjPSJodHRwOi8vcHVybC5vcmcvZGMvZWxlbWVudHMvMS4xLyIKICAgIHhtbG5zOkdJTVA9Imh0dHA6Ly93d3cuZ2ltcC5vcmcveG1wLyIKICAgIHhtbG5zOnRpZmY9Imh0dHA6Ly9ucy5hZG9iZS5jb20vdGlmZi8xLjAvIgogICAgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIgogICB4bXBNTTpEb2N1bWVudElEPSJnaW1wOmRvY2lkOmdpbXA6NmM4YjkxNDEtYTEwOS00ODc5LWE1NzMtNTdkNDRmNDI0ODlmIgogICB4bXBNTTpJbnN0YW5jZUlEPSJ4bXAuaWlkOjBmMzc0MjkxLWViNGMtNGZmZi1hNWFhLTE1ZmNmZjhkYmIxNyIKICAgeG1wTU06T3JpZ2luYWxEb2N1bWVudElEPSJ4bXAuZGlkOjY2ODdlMjU0LTExM2ItNDdiOC04ODg0LTZjYWJiODc2NjNkZCIKICAgZGM6Rm9ybWF0PSJpbWFnZS9wbmciCiAgIEdJTVA6QVBJPSIyLjAiCiAgIEdJTVA6UGxhdGZvcm09IkxpbnV4IgogICBHSU1QOlRpbWVTdGFtcD0iMTY1Njk2NTM2NzQyMDIyMCIKICAgR0lNUDpWZXJzaW9uPSIyLjEwLjMwIgogICB0aWZmOk9yaWVudGF0aW9uPSIxIgogICB4bXA6Q3JlYXRvclRvb2w9IkdJTVAgMi4xMCI+CiAgIDx4bXBNTTpIaXN0b3J5PgogICAgPHJkZjpTZXE+CiAgICAgPHJkZjpsaQogICAgICBzdEV2dDphY3Rpb249InNhdmVkIgogICAgICBzdEV2dDpjaGFuZ2VkPSIvIgogICAgICBzdEV2dDppbnN0YW5jZUlEPSJ4bXAuaWlkOmFlYzI2N2I5LTAyMzgtNGNmNy1iZTU4LWNmYjg0YjZhZjZmOSIKICAgICAgc3RFdnQ6c29mdHdhcmVBZ2VudD0iR2ltcCAyLjEwIChMaW51eCkiCiAgICAgIHN0RXZ0OndoZW49IjIwMjItMDctMDRUMjE6NTU6NDUrMDI6MDAiLz4KICAgICA8cmRmOmxpCiAgICAgIHN0RXZ0OmFjdGlvbj0ic2F2ZWQiCiAgICAgIHN0RXZ0OmNoYW5nZWQ9Ii8iCiAgICAgIHN0RXZ0Omluc3RhbmNlSUQ9InhtcC5paWQ6NDAzOGIxNGUtYWE4YS00ZTZhLTliMWItY2JmMjc2YjI3NTc0IgogICAgICBzdEV2dDpzb2Z0d2FyZUFnZW50PSJHaW1wIDIuMTAgKExpbnV4KSIKICAgICAgc3RFdnQ6d2hlbj0iMjAyMi0wNy0wNFQyMjowOToyNyswMjowMCIvPgogICAgPC9yZGY6U2VxPgogICA8L3htcE1NOkhpc3Rvcnk+CiAgPC9yZGY6RGVzY3JpcHRpb24+CiA8L3JkZjpSREY+CjwveDp4bXBtZXRhPgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAgCjw/eHBhY2tldCBlbmQ9InciPz7q/t4IAAAABmJLR0QA/wD/AP+gvaeTAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAB3RJTUUH5gcEFAkbcQNgcQAACGJJREFUeNrlm3+QVWUZxz/POXd/uMuPdtEEIdiR2VV+OAIS4gQYjpVUiJo2FRZWqFMpUeMkTJGD1ijmpAnZTNM4SfiHCWM/kBKiQvzRpCRiKGnZAlIYESiKi7v3vv1xvsc9u9y995y9596V3WdmZ5mX877v8zzv8z4/vs+70Dc0ArgDeAb4K7AKs/cxQGgssBdwwJv6ccCrmI0fCAr4hQS+y8wa8LyhwO0a+wNg/Vn4eqANeBHIRMY94M9AFji1kgx5FVaAD1QD+4COyHgO2CN+ruymnH5FVRL0DaA5Mt4EHNQ1CK6C2Wn9VQlfk5CvAN8GlgKtGrsH2KJ/vwSM6Y8K8IDvAm9FTrwdWCkLqQZ++I4SzEb0SzMwzxuN2XxgAWan51HSSilhC1DDAKRq4BEp4aaBqADM90cAB4A2zCYwQOmLsoL1tbW1NuCkz2QyPrANyOF5M1K3ssQe3OwMnJuuOD5G2d0piu2Hgf8qxL0IbAf+kQKflwHrgA3Ax4Ba4PvAMOA14F/ATjN7GudaXWAxqdLJwDIJlY2ErzCEtQHHuo07KaUxSaZoZi3A9cBDKprmKyrs0F7jpPBDefbr0HdLxHMqMfvLwP+0QRuwCbMbdRLjgdOARjzvVKAFOE/p7HLgm8XSWjMzYApwK/C3iDA54ABml+jTazX+vYAz72SgBbMpspCbga1SkpMlXltKul8F3Bc5yVskZFrUCFwH/EXCOiVHG4EbgCmYRZU3DDii61VVYN3RmN0OHNWa9xb5vkdaoQVeANILQZ7XhNnd8hcOeBv4PbBQpl2I1mnO+TF2mijeHWZ3JGXyXN2nfcColARvAdboGjnF99swa06wyqc1N55AZqNUfGUxm5WE3V9qo8tTEH04sCqS++8BFlNTU9+LtYZpnR0JItil2veRJHfzqKqx3tflmUy1Kr/Dkervq5jVlajQJ+XoRsQzAvOB5xWlYvmwWZHStLd0npybk+O6CRiSkhcJ4bNLE8wJC6sL4yBCQ/X71V7k7rVicAswWbF8kkLi6ykp4E/6PT3BnJciwEtXQ00RLhvvstnVwDnAfmCRV1OzNnfsWNpZ2TMKm5Pjn4zlcC6vTPmEPKLf70nA1HzdzXOAX+N5ZwMPlkF4fN/fLb8yLvYk50JkaW8cpzFcTubxGJ42o8wsCxzFbBGVAFrNnlQ4jZvu/lZWM7Z4+ldVZfKabyns9ITo1NOJ8b8CzKBytEYJzsQY3w5WwdSa78ofd1rt7e0OWK+K64oeFm1wudwmYB6wHbNzgccqqIC9Mu04SdpcRaCH6QrFF6QJ+vhZBg3yuwuv++6A30SiRiVpsfb/XOGzH2ziNQdMi5+11tebam8Xqca6C/+QrKQv6FPi4RtFfMVsCb9VVWci+oCsIIy7JxH07tISvgH4hIQZlnDuR8XHdwqEiyohSVnMPtxbJq8HPqtosEabbk5B+IlCccLafz+eNzXB/Jmad3eB018UYolpmNwNWmwnZg0prPe7PGjO0/i+paSA8cIwXsdsbGmsBh7+mCCoM9LIZejaFXIRtCmTggLqhUU6AS4l0SCBCjnd1VTSGOC5PArYTSbjlagAA36m/1uXRlJ2sxZbAxhm04AFlP6IYS5dQdR24Yil+oAGlfI7EqbyeWkMwfOV/wDv1djPZQ23pKCE9wN3AndiljSL/JAUcGue/5uM2SkpGKr9WJt8PTJ6prKwtJRQah5wY7k2GK7T36dcOlrzT4go4TY5tUrTV6SAheVONXvqyoaW4ID78LxKZ4TLtfcl5dpgq064ucAVaQJ2ipHHgJEVVMC92ndqORYfKvN/ttgdN7NGOnv4u4EPVkgBmwn6CWV5RzRFAv00VuFUU5PBbIXqhreBZZbJVJVReA/4t6JTWfzPXClgecJ5l4kpBzxBmt2k4x10h65pWehiCbGVoMs6MfbM4L3vRjqfwS5TFZkmXaT1V8YsupYQPMq6UmlyQToT2MXx7e+lCWK+D1xN0J11BB3fuaSHFX5L6y4okm4vpbNTHP7skox5KUMn5PwAMA+zLxFA3DnMLkrI6EjgfgLANEfQAJ2ZQvK0iWIAZ8BrDtgvGeYBD2psW09F1/nS0sN+16LkAo1v6CXDMwkQ5rDnvwmYZdXVyRVhVk8Aib9cxAFu0H4XvDO1ttYTLthDd9ns83lLyABZOUCAqpZSAV6hIiVUxBPAZ4C6BAoIa4CfFPmyFTiA53WPRtdp/hfyTZoTqfyidJbM+NGi/HleI4Xb6VUyx8fpfGqzn+C9z6SiYc1sHkHjphjE9aigsLO6jYeo1px8k06S5rIqdJq1UZjtXRUT6TmI5w0pIogR9PZWE/QMQyf1AkFvcXYByxhchIdqWYjDbJdkaJZMWcnYY2SaTeer7VyEsdUxk467EigrWsMvVEbZFtnzCAEIu0JXZZoyvzpdqfBnCGbNQq9/EKlRoo+5QlkOdn8ocbwj8v2RZLPXyCRfw2wtsB7ncjHu6Nk4t02edjpJn6sFdfyFOPcROapR3Tx2mG12dDvx6kiYfQP4FbACsyacu1zp/XZZxt7CCigdR9iMc7OVsGwsMeUdpWRmEnA6MFrgTPQaHCRAmJ8DnlICd4g+I7OP6+T/SOX/IuVdQb68cA6zi1NQ6HRgbbkqv3LRDKWhrSWBk75fR9CpzgmeP6HoRz3kFUlolda4nxPtz+nM9wcph8gRdJaS0jUKZf8kpXe/feEQxxG8M+4gADLjnuJCzTmM2VROcJqpAiZH8PSu0HuCOoInNx3Am5jN6R9xIXjR/TKdL0WXAC34fq2ZVSvGLwb+rm/2EbTn+xEFAOo9dDZGs7KMQ5G0tZ3ghfrwirHVB6poAj6pWj18vrZHFeIDQpAqRv8HfVFbRrIj1o4AAAAASUVORK5CYII=" + mediatype: "image/png" + install: + spec: + deployments: null + strategy: "" + installModes: + - supported: false + type: OwnNamespace + - supported: false + type: SingleNamespace + - supported: false + type: MultiNamespace + - supported: true + type: AllNamespaces + keywords: + - oot + - out-of-tree + - kernel + - module + - modules + links: + - name: GitHub repository + url: https://github.com/kubernetes-sigs/kernel-module-management + maintainers: + - email: quba@redhat.com + name: Quentin Barrand + maturity: alpha + provider: + name: Community + url: https://github.com/kubernetes-sigs/kernel-module-management + version: 0.0.0 diff --git a/config/manifests/kustomization.yaml b/config/manifests/kustomization.yaml new file mode 100644 index 000000000..d8dde601e --- /dev/null +++ b/config/manifests/kustomization.yaml @@ -0,0 +1,27 @@ +# These resources constitute the fully configured set of manifests +# used to generate the 'manifests/' directory in a bundle. +resources: +- bases/kmm-operator.clusterserviceversion.yaml +- ../default +- ../samples +- ../scorecard + +# [WEBHOOK] To enable webhooks, uncomment all the sections with [WEBHOOK] prefix. +# Do NOT uncomment sections with prefix [CERTMANAGER], as OLM does not support cert-manager. +# These patches remove the unnecessary "cert" volume and its manager container volumeMount. +#patchesJson6902: +#- target: +# group: apps +# version: v1 +# kind: Deployment +# name: controller-manager +# namespace: system +# patch: |- +# # Remove the manager container's "cert" volumeMount, since OLM will create and mount a set of certs. +# # Update the indices in this path if adding or removing containers/volumeMounts in the manager's Deployment. +# - op: remove +# path: /spec/template/spec/containers/1/volumeMounts/0 +# # Remove the "cert" volume, since OLM will create and mount a set of certs. +# # Update the indices in this path if adding or removing volumes in the manager's Deployment. +# - op: remove +# path: /spec/template/spec/volumes/0 diff --git a/config/prometheus/kustomization.yaml b/config/prometheus/kustomization.yaml new file mode 100644 index 000000000..d5ebb906c --- /dev/null +++ b/config/prometheus/kustomization.yaml @@ -0,0 +1,5 @@ +resources: +- monitor.yaml + + # in case of OCP, a openshift.io/cluster-monitoring: "true" + # must be added to the namespace of KMMO operator diff --git a/config/prometheus/monitor.yaml b/config/prometheus/monitor.yaml new file mode 100644 index 000000000..29e210a98 --- /dev/null +++ b/config/prometheus/monitor.yaml @@ -0,0 +1,49 @@ + +# Prometheus Monitor Service (Metrics) +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + control-plane: controller-manager + name: controller-manager-metrics-monitor + namespace: system +spec: + endpoints: + - path: /metrics + port: https + scheme: https + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + tlsConfig: + insecureSkipVerify: true + selector: + matchLabels: + control-plane: controller-manager +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: prometheus-k8s +rules: + - apiGroups: + - "" + resources: + - services + - endpoints + - pods + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: prometheus-k8s +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: prometheus-k8s +subjects: + - kind: ServiceAccount + name: prometheus-k8s + namespace: openshift-monitoring diff --git a/config/rbac/auth_proxy_client_clusterrole.yaml b/config/rbac/auth_proxy_client_clusterrole.yaml new file mode 100644 index 000000000..51a75db47 --- /dev/null +++ b/config/rbac/auth_proxy_client_clusterrole.yaml @@ -0,0 +1,9 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-reader +rules: +- nonResourceURLs: + - "/metrics" + verbs: + - get diff --git a/config/rbac/auth_proxy_role.yaml b/config/rbac/auth_proxy_role.yaml new file mode 100644 index 000000000..80e1857c5 --- /dev/null +++ b/config/rbac/auth_proxy_role.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: proxy-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create diff --git a/config/rbac/auth_proxy_role_binding.yaml b/config/rbac/auth_proxy_role_binding.yaml new file mode 100644 index 000000000..ec7acc0a1 --- /dev/null +++ b/config/rbac/auth_proxy_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: proxy-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: proxy-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/rbac/auth_proxy_service.yaml b/config/rbac/auth_proxy_service.yaml new file mode 100644 index 000000000..71f179727 --- /dev/null +++ b/config/rbac/auth_proxy_service.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + control-plane: controller-manager + name: controller-manager-metrics-service + namespace: system +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: https + selector: + control-plane: controller-manager diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml new file mode 100644 index 000000000..731832a6a --- /dev/null +++ b/config/rbac/kustomization.yaml @@ -0,0 +1,18 @@ +resources: +# All RBAC will be applied under this service account in +# the deployment namespace. You may comment out this resource +# if your manager will use a service account that exists at +# runtime. Be sure to update RoleBinding and ClusterRoleBinding +# subjects if changing service account names. +- service_account.yaml +- role.yaml +- role_binding.yaml +- leader_election_role.yaml +- leader_election_role_binding.yaml +# Comment the following 4 lines if you want to disable +# the auth proxy (https://github.com/brancz/kube-rbac-proxy) +# which protects your /metrics endpoint. +- auth_proxy_service.yaml +- auth_proxy_role.yaml +- auth_proxy_role_binding.yaml +- auth_proxy_client_clusterrole.yaml diff --git a/config/rbac/leader_election_role.yaml b/config/rbac/leader_election_role.yaml new file mode 100644 index 000000000..4190ec805 --- /dev/null +++ b/config/rbac/leader_election_role.yaml @@ -0,0 +1,37 @@ +# permissions to do leader election. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: leader-election-role +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch diff --git a/config/rbac/leader_election_role_binding.yaml b/config/rbac/leader_election_role_binding.yaml new file mode 100644 index 000000000..1d1321ed4 --- /dev/null +++ b/config/rbac/leader_election_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: leader-election-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: leader-election-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/rbac/module_editor_role.yaml b/config/rbac/module_editor_role.yaml new file mode 100644 index 000000000..0cdacddaa --- /dev/null +++ b/config/rbac/module_editor_role.yaml @@ -0,0 +1,24 @@ +# permissions for end users to edit modules. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: module-editor-role +rules: +- apiGroups: + - kmm.sigs.k8s.io + resources: + - modules + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kmm.sigs.k8s.io + resources: + - modules/status + verbs: + - get diff --git a/config/rbac/module_viewer_role.yaml b/config/rbac/module_viewer_role.yaml new file mode 100644 index 000000000..d4516525c --- /dev/null +++ b/config/rbac/module_viewer_role.yaml @@ -0,0 +1,20 @@ +# permissions for end users to view modules. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: module-viewer-role +rules: +- apiGroups: + - kmm.sigs.k8s.io + resources: + - modules + verbs: + - get + - list + - watch +- apiGroups: + - kmm.sigs.k8s.io + resources: + - modules/status + verbs: + - get diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml new file mode 100644 index 000000000..5b215bf2b --- /dev/null +++ b/config/rbac/role.yaml @@ -0,0 +1,98 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: manager-role +rules: +- apiGroups: + - apps + resources: + - daemonsets + verbs: + - create + - delete + - get + - list + - patch + - watch +- apiGroups: + - batch + resources: + - jobs + verbs: + - create + - list + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - patch + - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - patch + - watch +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch +- apiGroups: + - kmm.sigs.k8s.io + resources: + - modules + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kmm.sigs.k8s.io + resources: + - modules/finalizers + verbs: + - update +- apiGroups: + - kmm.sigs.k8s.io + resources: + - modules/status + verbs: + - get + - patch + - update +- apiGroups: + - kmm.sigs.k8s.io + resources: + - preflightvalidations + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - kmm.sigs.k8s.io + resources: + - preflightvalidations/status + verbs: + - get + - patch + - update diff --git a/config/rbac/role_binding.yaml b/config/rbac/role_binding.yaml new file mode 100644 index 000000000..2070ede44 --- /dev/null +++ b/config/rbac/role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: manager-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/config/rbac/service_account.yaml b/config/rbac/service_account.yaml new file mode 100644 index 000000000..7cd6025bf --- /dev/null +++ b/config/rbac/service_account.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: controller-manager + namespace: system diff --git a/config/samples/kmm_v1alpha1_module.yaml b/config/samples/kmm_v1alpha1_module.yaml new file mode 100644 index 000000000..563578959 --- /dev/null +++ b/config/samples/kmm_v1alpha1_module.yaml @@ -0,0 +1,6 @@ +apiVersion: kmm.sigs.k8s.io/v1beta1 +kind: Module +metadata: + name: module-sample +spec: + # TODO(user): Add fields here diff --git a/config/samples/kustomization.yaml b/config/samples/kustomization.yaml new file mode 100644 index 000000000..ce973088a --- /dev/null +++ b/config/samples/kustomization.yaml @@ -0,0 +1,4 @@ +## Append samples you want in your CSV to this file as resources ## +resources: +- kmm_v1alpha1_module.yaml +#+kubebuilder:scaffold:manifestskustomizesamples diff --git a/config/scorecard/bases/config.yaml b/config/scorecard/bases/config.yaml new file mode 100644 index 000000000..c77047841 --- /dev/null +++ b/config/scorecard/bases/config.yaml @@ -0,0 +1,7 @@ +apiVersion: scorecard.operatorframework.io/v1alpha3 +kind: Configuration +metadata: + name: config +stages: +- parallel: true + tests: [] diff --git a/config/scorecard/kustomization.yaml b/config/scorecard/kustomization.yaml new file mode 100644 index 000000000..50cd2d084 --- /dev/null +++ b/config/scorecard/kustomization.yaml @@ -0,0 +1,16 @@ +resources: +- bases/config.yaml +patchesJson6902: +- path: patches/basic.config.yaml + target: + group: scorecard.operatorframework.io + version: v1alpha3 + kind: Configuration + name: config +- path: patches/olm.config.yaml + target: + group: scorecard.operatorframework.io + version: v1alpha3 + kind: Configuration + name: config +#+kubebuilder:scaffold:patchesJson6902 diff --git a/config/scorecard/patches/basic.config.yaml b/config/scorecard/patches/basic.config.yaml new file mode 100644 index 000000000..b866bd116 --- /dev/null +++ b/config/scorecard/patches/basic.config.yaml @@ -0,0 +1,10 @@ +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - basic-check-spec + image: quay.io/operator-framework/scorecard-test:v1.18.0 + labels: + suite: basic + test: basic-check-spec-test diff --git a/config/scorecard/patches/olm.config.yaml b/config/scorecard/patches/olm.config.yaml new file mode 100644 index 000000000..0c060e8ab --- /dev/null +++ b/config/scorecard/patches/olm.config.yaml @@ -0,0 +1,50 @@ +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-bundle-validation + image: quay.io/operator-framework/scorecard-test:v1.18.0 + labels: + suite: olm + test: olm-bundle-validation-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-crds-have-validation + image: quay.io/operator-framework/scorecard-test:v1.18.0 + labels: + suite: olm + test: olm-crds-have-validation-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-crds-have-resources + image: quay.io/operator-framework/scorecard-test:v1.18.0 + labels: + suite: olm + test: olm-crds-have-resources-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-spec-descriptors + image: quay.io/operator-framework/scorecard-test:v1.18.0 + labels: + suite: olm + test: olm-spec-descriptors-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-status-descriptors + image: quay.io/operator-framework/scorecard-test:v1.18.0 + labels: + suite: olm + test: olm-status-descriptors-test diff --git a/controllers/module_reconciler.go b/controllers/module_reconciler.go new file mode 100644 index 000000000..e49735274 --- /dev/null +++ b/controllers/module_reconciler.go @@ -0,0 +1,350 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + + kmmv1beta1 "github.com/kubernetes-sigs/kernel-module-management/api/v1beta1" + "github.com/kubernetes-sigs/kernel-module-management/internal/build" + "github.com/kubernetes-sigs/kernel-module-management/internal/daemonset" + "github.com/kubernetes-sigs/kernel-module-management/internal/filter" + "github.com/kubernetes-sigs/kernel-module-management/internal/metrics" + "github.com/kubernetes-sigs/kernel-module-management/internal/module" + "github.com/kubernetes-sigs/kernel-module-management/internal/statusupdater" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +// ModuleReconciler reconciles a Module object +type ModuleReconciler struct { + client.Client + + buildAPI build.Manager + daemonAPI daemonset.DaemonSetCreator + kernelAPI module.KernelMapper + metricsAPI metrics.Metrics + filter *filter.Filter + statusUpdaterAPI statusupdater.ModuleStatusUpdater +} + +func NewModuleReconciler( + client client.Client, + buildAPI build.Manager, + daemonAPI daemonset.DaemonSetCreator, + kernelAPI module.KernelMapper, + metricsAPI metrics.Metrics, + filter *filter.Filter, + statusUpdaterAPI statusupdater.ModuleStatusUpdater) *ModuleReconciler { + return &ModuleReconciler{ + Client: client, + buildAPI: buildAPI, + daemonAPI: daemonAPI, + kernelAPI: kernelAPI, + metricsAPI: metricsAPI, + filter: filter, + statusUpdaterAPI: statusUpdaterAPI, + } +} + +//+kubebuilder:rbac:groups=kmm.sigs.k8s.io,resources=modules,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=kmm.sigs.k8s.io,resources=modules/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=kmm.sigs.k8s.io,resources=modules/finalizers,verbs=update +//+kubebuilder:rbac:groups=apps,resources=daemonsets,verbs=create;delete;get;list;patch;watch +//+kubebuilder:rbac:groups="core",resources=nodes,verbs=get;list;watch +//+kubebuilder:rbac:groups="core",resources=secrets,verbs=get;list;watch +//+kubebuilder:rbac:groups="batch",resources=jobs,verbs=create;list;watch + +// Reconcile lists all nodes and looks for kernels that match its mappings. +// For each mapping that matches at least one node in the cluster, it creates a DaemonSet running the container image +// on the nodes with a compatible kernel. +func (r *ModuleReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + res := ctrl.Result{} + + logger := log.FromContext(ctx) + + mod, err := r.getRequestedModule(ctx, req.NamespacedName) + if err != nil { + if k8serrors.IsNotFound(err) { + logger.Info("Module deleted") + return ctrl.Result{}, nil + } + + return res, fmt.Errorf("failed to get the requested %s KMMO CR: %w", req.NamespacedName, err) + } + + r.setKMMOMetrics(ctx) + + targetedNodes, err := r.getNodesListBySelector(ctx, mod) + if err != nil { + return res, fmt.Errorf("could get targeted nodes for module %s: %w", mod.Name, err) + } + + mappings, nodesWithMapping, err := r.getRelevantKernelMappingsAndNodes(ctx, mod, targetedNodes) + if err != nil { + return res, fmt.Errorf("could get kernel mappings and nodes for modules %s: %w", mod.Name, err) + } + + dsByKernelVersion, err := r.daemonAPI.ModuleDaemonSetsByKernelVersion(ctx, mod.Name, mod.Namespace) + if err != nil { + return res, fmt.Errorf("could get DaemonSets for module %s: %v", mod.Name, err) + } + + for kernelVersion, m := range mappings { + requeue, err := r.handleBuild(ctx, mod, m, kernelVersion) + if err != nil { + return res, fmt.Errorf("failed to handle build for kernel version %s: %w", kernelVersion, err) + } + if requeue { + logger.Info("Build requires a requeue; skipping handling driver container for now", "kernelVersion", kernelVersion, "image", m) + res.Requeue = true + continue + } + + err = r.handleDriverContainer(ctx, mod, m, dsByKernelVersion, kernelVersion) + if err != nil { + return res, fmt.Errorf("failed to handle driver container for kernel version %s: %v", kernelVersion, err) + } + } + + logger.Info("Handle device plugin") + err = r.handleDevicePlugin(ctx, mod) + if err != nil { + return res, fmt.Errorf("could handle device plugin: %w", err) + } + + logger.Info("Garbage-collecting DaemonSets") + + // Garbage collect old DaemonSets for which there are no nodes. + validKernels := sets.StringKeySet(mappings) + + deleted, err := r.daemonAPI.GarbageCollect(ctx, dsByKernelVersion, validKernels) + if err != nil { + return res, fmt.Errorf("could not garbage collect DaemonSets: %v", err) + } + + err = r.statusUpdaterAPI.ModuleUpdateStatus(ctx, mod, nodesWithMapping, targetedNodes, dsByKernelVersion) + if err != nil { + return res, fmt.Errorf("failed to update status of the module: %w", err) + } + + logger.Info("Garbage-collected DaemonSets", "names", deleted) + + return res, nil +} + +func (r *ModuleReconciler) getRelevantKernelMappingsAndNodes(ctx context.Context, + mod *kmmv1beta1.Module, + targetedNodes []v1.Node) (map[string]*kmmv1beta1.KernelMapping, []v1.Node, error) { + + mappings := make(map[string]*kmmv1beta1.KernelMapping) + logger := log.FromContext(ctx) + + nodes := make([]v1.Node, 0, len(targetedNodes)) + + for _, node := range targetedNodes { + osConfig := r.kernelAPI.GetNodeOSConfig(&node) + kernelVersion := node.Status.NodeInfo.KernelVersion + + nodeLogger := logger.WithValues( + "node", node.Name, + "kernel version", kernelVersion, + ) + + if image, ok := mappings[kernelVersion]; ok { + nodes = append(nodes, node) + nodeLogger.V(1).Info("Using cached image", "image", image) + continue + } + + m, err := r.kernelAPI.FindMappingForKernel(mod.Spec.ModuleLoader.Container.KernelMappings, kernelVersion) + if err != nil { + nodeLogger.Info("no suitable container image found; skipping node") + continue + } + + m, err = r.kernelAPI.PrepareKernelMapping(m, osConfig) + if err != nil { + nodes = append(nodes, node) + nodeLogger.Info("failed to substitute the template variables in the mapping", "error", err) + continue + } + + nodeLogger.V(1).Info("Found a valid mapping", + "image", m.ContainerImage, + "build", m.Build != nil, + ) + + mappings[kernelVersion] = m + nodes = append(nodes, node) + } + return mappings, nodes, nil +} + +func (r *ModuleReconciler) getNodesListBySelector(ctx context.Context, mod *kmmv1beta1.Module) ([]v1.Node, error) { + logger := log.FromContext(ctx) + logger.V(1).Info("Listing nodes", "selector", mod.Spec.Selector) + + nodes := v1.NodeList{} + opt := client.MatchingLabels(mod.Spec.Selector) + if err := r.Client.List(ctx, &nodes, opt); err != nil { + logger.Error(err, "Could not list nodes") + return nil, fmt.Errorf("could not list nodes: %v", err) + } + return nodes.Items, nil +} + +func (r *ModuleReconciler) handleBuild(ctx context.Context, + mod *kmmv1beta1.Module, + km *kmmv1beta1.KernelMapping, + kernelVersion string) (bool, error) { + if mod.Spec.ModuleLoader.Container.Build == nil && km.Build == nil { + return false, nil + } + + // [TODO] check access to the image - execute build only if needed (image is inaccessible) + logger := log.FromContext(ctx).WithValues("kernel version", kernelVersion, "image", km.ContainerImage) + buildCtx := log.IntoContext(ctx, logger) + + buildRes, err := r.buildAPI.Sync(buildCtx, *mod, *km, kernelVersion) + if err != nil { + return false, fmt.Errorf("could not synchronize the build: %w", err) + } + + switch buildRes.Status { + case build.StatusCreated: + r.metricsAPI.SetCompletedStage(mod.Name, mod.Namespace, kernelVersion, metrics.BuildStage, false) + case build.StatusCompleted: + r.metricsAPI.SetCompletedStage(mod.Name, mod.Namespace, kernelVersion, metrics.BuildStage, true) + } + + return buildRes.Requeue, nil +} + +func (r *ModuleReconciler) handleDriverContainer(ctx context.Context, + mod *kmmv1beta1.Module, + km *kmmv1beta1.KernelMapping, + dsByKernelVersion map[string]*appsv1.DaemonSet, + kernelVersion string) error { + ds := &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Namespace: mod.Namespace}, + } + + logger := log.FromContext(ctx) + if existingDS := dsByKernelVersion[kernelVersion]; existingDS != nil { + logger.Info("updating existing driver container DS", "kernel version", kernelVersion, "image", km, "name", ds.Name) + ds = existingDS + } else { + logger.Info("creating new driver container DS", "kernel version", kernelVersion, "image", km) + ds.GenerateName = mod.Name + "-" + } + + opRes, err := controllerutil.CreateOrPatch(ctx, r.Client, ds, func() error { + return r.daemonAPI.SetDriverContainerAsDesired(ctx, ds, km.ContainerImage, *mod, kernelVersion) + }) + + if err == nil { + if opRes == controllerutil.OperationResultCreated { + r.metricsAPI.SetCompletedStage(mod.Name, mod.Namespace, kernelVersion, metrics.ModuleLoaderStage, false) + } + logger.Info("Reconciled Driver Container", "name", ds.Name, "result", opRes) + } + + return err +} + +func (r *ModuleReconciler) handleDevicePlugin(ctx context.Context, mod *kmmv1beta1.Module) error { + if mod.Spec.DevicePlugin == nil { + return nil + } + + logger := log.FromContext(ctx) + ds := &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{Namespace: mod.Namespace}, + } + name := mod.Name + "-device-plugin" + ds.Name = name + err := r.Client.Get(ctx, types.NamespacedName{Name: name, Namespace: mod.Namespace}, ds) + if err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to get the device plugin daemonset %s/%s: %w", name, mod.Namespace, err) + } + + opRes, err := controllerutil.CreateOrPatch(ctx, r.Client, ds, func() error { + return r.daemonAPI.SetDevicePluginAsDesired(ctx, ds, mod) + }) + + if err == nil { + if opRes == controllerutil.OperationResultCreated { + r.metricsAPI.SetCompletedStage(mod.Name, mod.Namespace, "", metrics.DevicePluginStage, false) + } + logger.Info("Reconciled Device Plugin", "name", ds.Name, "result", opRes) + } + + return err +} + +func (r *ModuleReconciler) setKMMOMetrics(ctx context.Context) { + logger := log.FromContext(ctx) + + mods := kmmv1beta1.ModuleList{} + err := r.Client.List(ctx, &mods) + if err != nil { + logger.V(1).Info("failed to list KMMomodules for metrics", "error", err) + } + + r.metricsAPI.SetExistingKMMOModules(len(mods.Items)) +} + +func (r *ModuleReconciler) getRequestedModule(ctx context.Context, namespacedName types.NamespacedName) (*kmmv1beta1.Module, error) { + mod := kmmv1beta1.Module{} + + if err := r.Client.Get(ctx, namespacedName, &mod); err != nil { + return nil, fmt.Errorf("failed to get the kmmo module %s: %w", namespacedName, err) + } + return &mod, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *ModuleReconciler) SetupWithManager(mgr ctrl.Manager, kernelLabel string) error { + return ctrl.NewControllerManagedBy(mgr). + For(&kmmv1beta1.Module{}). + Owns(&appsv1.DaemonSet{}). + Owns(&batchv1.Job{}). + Watches( + &source.Kind{Type: &v1.Node{}}, + handler.EnqueueRequestsFromMapFunc(r.filter.FindModulesForNode), + builder.WithPredicates( + r.filter.ModuleReconcilerNodePredicate(kernelLabel), + ), + ). + Named("module"). + Complete(r) +} diff --git a/controllers/module_reconciler_test.go b/controllers/module_reconciler_test.go new file mode 100644 index 000000000..9661208d1 --- /dev/null +++ b/controllers/module_reconciler_test.go @@ -0,0 +1,456 @@ +package controllers + +import ( + "context" + + "github.com/golang/mock/gomock" + kmmv1beta1 "github.com/kubernetes-sigs/kernel-module-management/api/v1beta1" + "github.com/kubernetes-sigs/kernel-module-management/internal/build" + "github.com/kubernetes-sigs/kernel-module-management/internal/client" + "github.com/kubernetes-sigs/kernel-module-management/internal/daemonset" + "github.com/kubernetes-sigs/kernel-module-management/internal/metrics" + "github.com/kubernetes-sigs/kernel-module-management/internal/module" + "github.com/kubernetes-sigs/kernel-module-management/internal/statusupdater" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +const ( + namespace = "namespace" +) + +var _ = Describe("ModuleReconciler", func() { + Describe("Reconcile", func() { + var ( + ctrl *gomock.Controller + clnt *client.MockClient + mockBM *build.MockManager + mockDC *daemonset.MockDaemonSetCreator + mockKM *module.MockKernelMapper + mockMetrics *metrics.MockMetrics + mockSU *statusupdater.MockModuleStatusUpdater + ) + + BeforeEach(func() { + ctrl = gomock.NewController(GinkgoT()) + clnt = client.NewMockClient(ctrl) + mockBM = build.NewMockManager(ctrl) + mockDC = daemonset.NewMockDaemonSetCreator(ctrl) + mockKM = module.NewMockKernelMapper(ctrl) + mockMetrics = metrics.NewMockMetrics(ctrl) + mockSU = statusupdater.NewMockModuleStatusUpdater(ctrl) + }) + + const moduleName = "test-module" + + nsn := types.NamespacedName{ + Name: moduleName, + Namespace: namespace, + } + + req := reconcile.Request{NamespacedName: nsn} + + ctx := context.Background() + + It("should do nothing if the Module is not available anymore", func() { + clnt. + EXPECT(). + Get(ctx, nsn, &kmmv1beta1.Module{}). + Return( + apierrors.NewNotFound(schema.GroupResource{}, moduleName), + ) + + mr := NewModuleReconciler(clnt, mockBM, mockDC, mockKM, mockMetrics, nil, mockSU) + Expect( + mr.Reconcile(ctx, req), + ).To( + Equal(reconcile.Result{}), + ) + }) + + It("should do nothing when no nodes match the selector", func() { + mod := kmmv1beta1.Module{ + ObjectMeta: metav1.ObjectMeta{ + Name: moduleName, + Namespace: namespace, + }, + Spec: kmmv1beta1.ModuleSpec{ + Selector: map[string]string{"key": "value"}, + }, + } + + gomock.InOrder( + clnt.EXPECT().Get(ctx, req.NamespacedName, gomock.Any()).DoAndReturn( + func(_ interface{}, _ interface{}, m *kmmv1beta1.Module) error { + m.ObjectMeta = mod.ObjectMeta + m.Spec = mod.Spec + return nil + }, + ), + clnt.EXPECT().List(ctx, gomock.Any(), gomock.Any()).DoAndReturn( + func(_ interface{}, list *kmmv1beta1.ModuleList, _ ...interface{}) error { + list.Items = []kmmv1beta1.Module{mod} + return nil + }, + ), + mockMetrics.EXPECT().SetExistingKMMOModules(1), + clnt.EXPECT().List(ctx, gomock.Any(), gomock.Any()).DoAndReturn( + func(_ interface{}, list *v1.NodeList, _ ...interface{}) error { + list.Items = []v1.Node{} + return nil + }, + ), + ) + + mr := NewModuleReconciler(clnt, mockBM, mockDC, mockKM, mockMetrics, nil, mockSU) + + dsByKernelVersion := make(map[string]*appsv1.DaemonSet) + + gomock.InOrder( + mockDC.EXPECT().ModuleDaemonSetsByKernelVersion(ctx, moduleName, namespace).Return(dsByKernelVersion, nil), + mockDC.EXPECT().GarbageCollect(ctx, dsByKernelVersion, sets.NewString()), + mockSU.EXPECT().ModuleUpdateStatus(ctx, &mod, []v1.Node{}, []v1.Node{}, dsByKernelVersion).Return(nil), + ) + + res, err := mr.Reconcile(context.Background(), req) + Expect(err).NotTo(HaveOccurred()) + Expect(res).To(Equal(reconcile.Result{})) + }) + + It("should remove obsolete DaemonSets when no nodes match the selector", func() { + const kernelVersion = "1.2.3" + + mod := kmmv1beta1.Module{ + ObjectMeta: metav1.ObjectMeta{ + Name: moduleName, + Namespace: namespace, + }, + Spec: kmmv1beta1.ModuleSpec{ + Selector: map[string]string{"key": "value"}, + }, + } + + ds := appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "some-daemonset", + Namespace: namespace, + }, + } + + gomock.InOrder( + clnt.EXPECT().Get(ctx, req.NamespacedName, gomock.Any()).DoAndReturn( + func(_ interface{}, _ interface{}, m *kmmv1beta1.Module) error { + m.ObjectMeta = mod.ObjectMeta + m.Spec = mod.Spec + return nil + }, + ), + clnt.EXPECT().List(ctx, gomock.Any(), gomock.Any()).DoAndReturn( + func(_ interface{}, list *kmmv1beta1.ModuleList, _ ...interface{}) error { + list.Items = []kmmv1beta1.Module{mod} + return nil + }, + ), + mockMetrics.EXPECT().SetExistingKMMOModules(1), + clnt.EXPECT().List(ctx, gomock.Any(), gomock.Any()).DoAndReturn( + func(_ interface{}, list *v1.NodeList, _ ...interface{}) error { + list.Items = []v1.Node{} + return nil + }, + ), + ) + + mr := NewModuleReconciler(clnt, mockBM, mockDC, mockKM, mockMetrics, nil, mockSU) + + dsByKernelVersion := map[string]*appsv1.DaemonSet{kernelVersion: &ds} + + gomock.InOrder( + mockDC.EXPECT().ModuleDaemonSetsByKernelVersion(ctx, moduleName, namespace).Return(dsByKernelVersion, nil), + mockDC.EXPECT().GarbageCollect(ctx, dsByKernelVersion, sets.NewString()), + mockSU.EXPECT().ModuleUpdateStatus(ctx, &mod, []v1.Node{}, []v1.Node{}, dsByKernelVersion).Return(nil), + ) + + res, err := mr.Reconcile(context.Background(), req) + Expect(err).NotTo(HaveOccurred()) + Expect(res).To(Equal(reconcile.Result{})) + }) + + It("should create a DaemonSet when a node matches the selector", func() { + const ( + imageName = "test-image" + kernelVersion = "1.2.3" + ) + + mappings := []kmmv1beta1.KernelMapping{ + { + ContainerImage: imageName, + Literal: kernelVersion, + }, + } + + osConfig := module.NodeOSConfig{} + + mod := kmmv1beta1.Module{ + ObjectMeta: metav1.ObjectMeta{ + Name: moduleName, + Namespace: namespace, + }, + Spec: kmmv1beta1.ModuleSpec{ + ModuleLoader: kmmv1beta1.ModuleLoaderSpec{ + Container: kmmv1beta1.ModuleLoaderContainerSpec{ + KernelMappings: mappings, + }, + }, + Selector: map[string]string{"key": "value"}, + }, + } + + nodeList := v1.NodeList{ + Items: []v1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Labels: map[string]string{"key": "value"}, + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{KernelVersion: kernelVersion}, + }, + }, + }, + } + + dsByKernelVersion := make(map[string]*appsv1.DaemonSet) + + mr := NewModuleReconciler(clnt, mockBM, mockDC, mockKM, mockMetrics, nil, mockSU) + + ds := appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: moduleName + "-", + Namespace: namespace, + }, + } + + gomock.InOrder( + clnt.EXPECT().Get(ctx, req.NamespacedName, gomock.Any()).DoAndReturn( + func(_ interface{}, _ interface{}, m *kmmv1beta1.Module) error { + m.ObjectMeta = mod.ObjectMeta + m.Spec = mod.Spec + return nil + }, + ), + clnt.EXPECT().List(ctx, gomock.Any(), gomock.Any()).DoAndReturn( + func(_ interface{}, list *kmmv1beta1.ModuleList, _ ...interface{}) error { + return nil + }, + ), + mockMetrics.EXPECT().SetExistingKMMOModules(0), + clnt.EXPECT().List(ctx, gomock.Any(), gomock.Any()).DoAndReturn( + func(_ interface{}, list *v1.NodeList, _ ...interface{}) error { + list.Items = nodeList.Items + return nil + }, + ), + mockKM.EXPECT().GetNodeOSConfig(&nodeList.Items[0]).Return(&osConfig), + mockKM.EXPECT().FindMappingForKernel(mappings, kernelVersion).Return(&mappings[0], nil), + mockKM.EXPECT().PrepareKernelMapping(&mappings[0], &osConfig).Return(&mappings[0], nil), + mockDC.EXPECT().ModuleDaemonSetsByKernelVersion(ctx, moduleName, namespace).Return(dsByKernelVersion, nil), + clnt.EXPECT().Get(ctx, gomock.Any(), gomock.Any()).Return(apierrors.NewNotFound(schema.GroupResource{}, "whatever")), + mockDC.EXPECT().SetDriverContainerAsDesired(context.Background(), &ds, imageName, gomock.AssignableToTypeOf(mod), kernelVersion), + clnt.EXPECT().Create(ctx, gomock.Any()).Return(nil), + mockMetrics.EXPECT().SetCompletedStage(moduleName, namespace, kernelVersion, metrics.ModuleLoaderStage, false), + mockDC.EXPECT().GarbageCollect(ctx, dsByKernelVersion, sets.NewString(kernelVersion)), + mockSU.EXPECT().ModuleUpdateStatus(ctx, &mod, nodeList.Items, nodeList.Items, dsByKernelVersion).Return(nil), + ) + + res, err := mr.Reconcile(context.Background(), req) + Expect(err).NotTo(HaveOccurred()) + Expect(res).To(Equal(reconcile.Result{})) + }) + + It("should patch the DaemonSet when it already exists", func() { + const ( + imageName = "test-image" + kernelVersion = "1.2.3" + ) + + osConfig := module.NodeOSConfig{} + + mappings := []kmmv1beta1.KernelMapping{ + { + ContainerImage: imageName, + Literal: kernelVersion, + }, + } + + nodeLabels := map[string]string{"key": "value"} + + mod := kmmv1beta1.Module{ + ObjectMeta: metav1.ObjectMeta{ + Name: moduleName, + Namespace: namespace, + }, + Spec: kmmv1beta1.ModuleSpec{ + ModuleLoader: kmmv1beta1.ModuleLoaderSpec{ + Container: kmmv1beta1.ModuleLoaderContainerSpec{ + KernelMappings: mappings, + }, + }, + Selector: nodeLabels, + }, + } + + nodeList := v1.NodeList{ + Items: []v1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + Labels: nodeLabels, + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{KernelVersion: kernelVersion}, + }, + }, + }, + } + + const ( + dsName = "some-daemonset" + dsNamespace = "test-namespace" + ) + + ds := appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: dsName, + Namespace: dsNamespace, + }, + } + + gomock.InOrder( + clnt.EXPECT().Get(ctx, req.NamespacedName, gomock.Any()).DoAndReturn( + func(_ interface{}, _ interface{}, m *kmmv1beta1.Module) error { + m.ObjectMeta = mod.ObjectMeta + m.Spec = mod.Spec + return nil + }, + ), + clnt.EXPECT().List(ctx, gomock.Any(), gomock.Any()).DoAndReturn( + func(_ interface{}, list *kmmv1beta1.ModuleList, _ ...interface{}) error { + list.Items = []kmmv1beta1.Module{mod} + return nil + }, + ), + mockMetrics.EXPECT().SetExistingKMMOModules(1), + clnt.EXPECT().List(ctx, gomock.Any(), gomock.Any()).DoAndReturn( + func(_ interface{}, list *v1.NodeList, _ ...interface{}) error { + list.Items = nodeList.Items + return nil + }, + ), + clnt.EXPECT().Get(ctx, gomock.Any(), gomock.Any()), + clnt.EXPECT().Patch(ctx, gomock.Any(), gomock.Any()), + ) + + mr := NewModuleReconciler(clnt, mockBM, mockDC, mockKM, mockMetrics, nil, mockSU) + + dsByKernelVersion := map[string]*appsv1.DaemonSet{kernelVersion: &ds} + + gomock.InOrder( + mockKM.EXPECT().GetNodeOSConfig(&nodeList.Items[0]).Return(&osConfig), + mockKM.EXPECT().FindMappingForKernel(mappings, kernelVersion).Return(&mappings[0], nil), + mockKM.EXPECT().PrepareKernelMapping(&mappings[0], &osConfig).Return(&mappings[0], nil), + mockDC.EXPECT().ModuleDaemonSetsByKernelVersion(ctx, moduleName, namespace).Return(dsByKernelVersion, nil), + mockDC.EXPECT().SetDriverContainerAsDesired(context.Background(), &ds, imageName, gomock.AssignableToTypeOf(mod), kernelVersion).Do( + func(ctx context.Context, d *appsv1.DaemonSet, _ string, _ kmmv1beta1.Module, _ string) { + d.SetLabels(map[string]string{"test": "test"}) + }), + mockDC.EXPECT().GarbageCollect(ctx, dsByKernelVersion, sets.NewString(kernelVersion)), + mockSU.EXPECT().ModuleUpdateStatus(ctx, &mod, nodeList.Items, nodeList.Items, dsByKernelVersion).Return(nil), + ) + + res, err := mr.Reconcile(context.Background(), req) + Expect(err).NotTo(HaveOccurred()) + Expect(res).To(Equal(reconcile.Result{})) + }) + + It("should create a Device plugin if defined in the module", func() { + const ( + imageName = "test-image" + kernelVersion = "1.2.3" + ) + + mappings := []kmmv1beta1.KernelMapping{ + { + ContainerImage: imageName, + Literal: kernelVersion, + }, + } + + mod := kmmv1beta1.Module{ + ObjectMeta: metav1.ObjectMeta{ + Name: moduleName, + Namespace: namespace, + }, + Spec: kmmv1beta1.ModuleSpec{ + DevicePlugin: &kmmv1beta1.DevicePluginSpec{}, + ModuleLoader: kmmv1beta1.ModuleLoaderSpec{ + Container: kmmv1beta1.ModuleLoaderContainerSpec{ + KernelMappings: mappings, + }, + }, + Selector: map[string]string{"key": "value"}, + }, + } + + mr := NewModuleReconciler(clnt, mockBM, mockDC, mockKM, mockMetrics, nil, mockSU) + + ds := appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: moduleName + "-device-plugin", + Namespace: namespace, + }, + } + + gomock.InOrder( + clnt.EXPECT().Get(ctx, req.NamespacedName, gomock.Any()).DoAndReturn( + func(_ interface{}, _ interface{}, m *kmmv1beta1.Module) error { + m.ObjectMeta = mod.ObjectMeta + m.Spec = mod.Spec + return nil + }, + ), + clnt.EXPECT().List(ctx, gomock.Any(), gomock.Any()).DoAndReturn( + func(_ interface{}, list *kmmv1beta1.ModuleList, _ ...interface{}) error { + return nil + }, + ), + mockMetrics.EXPECT().SetExistingKMMOModules(0), + clnt.EXPECT().List(ctx, gomock.Any(), gomock.Any()).DoAndReturn( + func(_ interface{}, list *v1.NodeList, _ ...interface{}) error { + list.Items = []v1.Node{} + return nil + }, + ), + mockDC.EXPECT().ModuleDaemonSetsByKernelVersion(ctx, moduleName, namespace).Return(nil, nil), + clnt.EXPECT().Get(ctx, gomock.Any(), gomock.Any()).Return(apierrors.NewNotFound(schema.GroupResource{}, "whatever")), + clnt.EXPECT().Get(ctx, gomock.Any(), gomock.Any()).Return(apierrors.NewNotFound(schema.GroupResource{}, "whatever")), + mockDC.EXPECT().SetDevicePluginAsDesired(context.Background(), &ds, gomock.AssignableToTypeOf(&mod)), + clnt.EXPECT().Create(ctx, gomock.Any()).Return(nil), + mockMetrics.EXPECT().SetCompletedStage(moduleName, namespace, "", metrics.DevicePluginStage, false), + mockDC.EXPECT().GarbageCollect(ctx, nil, sets.NewString()), + mockSU.EXPECT().ModuleUpdateStatus(ctx, &mod, []v1.Node{}, []v1.Node{}, nil).Return(nil), + ) + + res, err := mr.Reconcile(context.Background(), req) + Expect(err).NotTo(HaveOccurred()) + Expect(res).To(Equal(reconcile.Result{})) + }) + }) +}) diff --git a/controllers/node_kernel_reconciler.go b/controllers/node_kernel_reconciler.go new file mode 100644 index 000000000..4dea855e0 --- /dev/null +++ b/controllers/node_kernel_reconciler.go @@ -0,0 +1,72 @@ +package controllers + +import ( + "context" + "fmt" + + "github.com/kubernetes-sigs/kernel-module-management/internal/filter" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +//+kubebuilder:rbac:groups="core",resources=nodes,verbs=get;patch;list;watch + +type NodeKernelReconciler struct { + client client.Client + labelName string + filter *filter.Filter +} + +func NewNodeKernelReconciler(client client.Client, labelName string, filter *filter.Filter) *NodeKernelReconciler { + return &NodeKernelReconciler{ + client: client, + labelName: labelName, + filter: filter, + } +} + +func (r *NodeKernelReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + node := v1.Node{} + + logger := log.FromContext(ctx) + + if err := r.client.Get(ctx, types.NamespacedName{Name: req.Name}, &node); err != nil { + return ctrl.Result{}, fmt.Errorf("could not get node: %v", err) + } + + kernelVersion := node.Status.NodeInfo.KernelVersion + + logger.Info( + "Patching node label", + "old kernel", node.Labels[r.labelName], + "new kernel", kernelVersion) + + p := client.MergeFrom(node.DeepCopy()) + + if node.Labels == nil { + node.Labels = make(map[string]string) + } + + node.Labels[r.labelName] = kernelVersion + + if err := r.client.Patch(ctx, &node, p); err != nil { + return ctrl.Result{}, fmt.Errorf("could not patch the node: %v", err) + } + + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *NodeKernelReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl. + NewControllerManagedBy(mgr). + Named("node-kernel"). + For(&v1.Node{}). + WithEventFilter( + r.filter.NodeKernelReconcilerPredicate(r.labelName), + ). + Complete(r) +} diff --git a/controllers/node_kernel_reconciler_test.go b/controllers/node_kernel_reconciler_test.go new file mode 100644 index 000000000..6e633ce25 --- /dev/null +++ b/controllers/node_kernel_reconciler_test.go @@ -0,0 +1,109 @@ +package controllers + +import ( + "context" + "errors" + + "github.com/golang/mock/gomock" + "github.com/kubernetes-sigs/kernel-module-management/internal/client" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + runtimectrl "sigs.k8s.io/controller-runtime" +) + +var _ = Describe("NodeKernelReconciler", func() { + Describe("Reconcile", func() { + var ( + gCtrl *gomock.Controller + clnt *client.MockClient + ) + BeforeEach(func() { + gCtrl = gomock.NewController(GinkgoT()) + clnt = client.NewMockClient(gCtrl) + }) + const ( + kernelVersion = "1.2.3" + labelName = "label-name" + nodeName = "node-name" + ) + + It("should return an error if the node cannot be found anymore", func() { + ctx := context.Background() + clnt.EXPECT().Get(ctx, gomock.Any(), gomock.Any()).Return(errors.New("some error")) + + nkr := NewNodeKernelReconciler(clnt, labelName, nil) + req := runtimectrl.Request{ + NamespacedName: types.NamespacedName{Name: nodeName}, + } + + _, err := nkr.Reconcile(ctx, req) + Expect(err).To(HaveOccurred()) + }) + + It("should set the label if it does not exist", func() { + node := v1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: nodeName}, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{KernelVersion: kernelVersion}, + }, + } + + ctx := context.Background() + gomock.InOrder( + clnt.EXPECT().Get(ctx, gomock.Any(), gomock.Any()).DoAndReturn( + func(_ interface{}, _ interface{}, n *v1.Node) error { + n.ObjectMeta = node.ObjectMeta + n.Status = node.Status + return nil + }, + ), + clnt.EXPECT().Patch(ctx, gomock.Any(), gomock.Any()), + ) + + nkr := NewNodeKernelReconciler(clnt, labelName, nil) + req := runtimectrl.Request{ + NamespacedName: types.NamespacedName{Name: nodeName}, + } + + res, err := nkr.Reconcile(ctx, req) + Expect(err).NotTo(HaveOccurred()) + Expect(res).To(Equal(res)) + }) + + It("should set the label if it already exists", func() { + node := v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + Labels: map[string]string{kernelVersion: "4.5.6"}, + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{KernelVersion: kernelVersion}, + }, + } + + ctx := context.Background() + gomock.InOrder( + clnt.EXPECT().Get(ctx, gomock.Any(), gomock.Any()).DoAndReturn( + func(_ interface{}, _ interface{}, n *v1.Node) error { + n.ObjectMeta = node.ObjectMeta + n.Status = node.Status + return nil + }, + ), + clnt.EXPECT().Patch(ctx, gomock.Any(), gomock.Any()), + ) + + nkr := NewNodeKernelReconciler(clnt, labelName, nil) + req := runtimectrl.Request{ + NamespacedName: types.NamespacedName{Name: nodeName}, + } + + res, err := nkr.Reconcile(ctx, req) + Expect(err).NotTo(HaveOccurred()) + Expect(res).To(Equal(res)) + }) + }) +}) diff --git a/controllers/pod_node_module_reconciler.go b/controllers/pod_node_module_reconciler.go new file mode 100644 index 000000000..e796cc102 --- /dev/null +++ b/controllers/pod_node_module_reconciler.go @@ -0,0 +1,148 @@ +package controllers + +import ( + "context" + "fmt" + + "github.com/kubernetes-sigs/kernel-module-management/internal/constants" + "github.com/kubernetes-sigs/kernel-module-management/internal/daemonset" + "github.com/kubernetes-sigs/kernel-module-management/internal/filter" + v1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/kubectl/pkg/util/podutils" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +//+kubebuilder:rbac:groups="core",resources=pods,verbs=get;patch;list;watch +//+kubebuilder:rbac:groups="core",resources=nodes,verbs=get;watch + +type PodNodeModuleReconciler struct { + client client.Client + daemonAPI daemonset.DaemonSetCreator +} + +func NewPodNodeModuleReconciler(client client.Client, daemonAPI daemonset.DaemonSetCreator) *PodNodeModuleReconciler { + return &PodNodeModuleReconciler{client: client, daemonAPI: daemonAPI} +} + +func (pnmr *PodNodeModuleReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + logger := ctrl.LoggerFrom(ctx) + + pod := v1.Pod{} + podNamespacedName := req.NamespacedName + + if err := pnmr.client.Get(ctx, podNamespacedName, &pod); err != nil { + if k8serrors.IsNotFound(err) { + logger.Info("Pod not found") + return ctrl.Result{}, nil + } + + return ctrl.Result{}, fmt.Errorf("could not get pod %s: %v", podNamespacedName, err) + } + + nodeName := pod.Spec.NodeName + + moduleName, ok := pod.Labels[constants.ModuleNameLabel] + if !ok { + return ctrl.Result{}, fmt.Errorf("pod %s has no %q label", podNamespacedName, constants.ModuleNameLabel) + } + + labelName := pnmr.daemonAPI.GetNodeLabelFromPod(&pod, moduleName) + + logger = logger.WithValues( + "node name", nodeName, + "module name", moduleName, + "label name", labelName, + ) + + if !podutils.IsPodReady(&pod) { + logger.Info("Unlabeling node") + + if err := pnmr.deleteLabel(ctx, nodeName, labelName); err != nil { + return ctrl.Result{}, fmt.Errorf("could not unlabel node %s: %v", nodeName, err) + } + + if !pod.DeletionTimestamp.IsZero() { + logger.Info("Pod deletion requested; removing finalizer") + + if err := pnmr.deleteFinalizer(ctx, &pod); err != nil { + return ctrl.Result{}, fmt.Errorf("could not delete the pod finalizer: %v", err) + } + } + + return ctrl.Result{}, nil + } + + logger.Info("Labeling node") + + if err := pnmr.addLabel(ctx, nodeName, labelName); err != nil { + return ctrl.Result{}, fmt.Errorf("could not label node %s with %q: %v", nodeName, labelName, err) + } + + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (pnmr *PodNodeModuleReconciler) SetupWithManager(mgr ctrl.Manager) error { + p := predicate.And( + predicate.Or( + filter.PodReadinessChangedPredicate( + mgr.GetLogger().WithName("pod-readiness-changed"), + ), + filter.DeletingPredicate(), + ), + filter.HasLabel(constants.ModuleNameLabel), + filter.PodHasSpecNodeName(), + ) + + return ctrl. + NewControllerManagedBy(mgr). + Named("pod-node-module"). + For(&v1.Pod{}). + WithEventFilter(p). + Complete(pnmr) +} + +func (pnmr *PodNodeModuleReconciler) addLabel(ctx context.Context, nodeName, labelName string) error { + node := v1.Node{} + + if err := pnmr.client.Get(ctx, types.NamespacedName{Name: nodeName}, &node); err != nil { + return fmt.Errorf("could not get node %s: %v", nodeName, err) + } + + nodeCopy := node.DeepCopy() + + if node.Labels == nil { + node.Labels = make(map[string]string, 1) + } + + node.Labels[labelName] = "" + + return pnmr.client.Patch(ctx, &node, client.MergeFrom(nodeCopy)) +} + +func (pnmr *PodNodeModuleReconciler) deleteFinalizer(ctx context.Context, pod *v1.Pod) error { + podCopy := pod.DeepCopy() + + controllerutil.RemoveFinalizer(pod, constants.NodeLabelerFinalizer) + + return pnmr.client.Patch(ctx, pod, client.MergeFrom(podCopy)) +} + +func (pnmr *PodNodeModuleReconciler) deleteLabel(ctx context.Context, nodeName, labelName string) error { + node := v1.Node{} + + if err := pnmr.client.Get(ctx, types.NamespacedName{Name: nodeName}, &node); err != nil { + return fmt.Errorf("could not get node %s: %v", nodeName, err) + } + + nodeCopy := node.DeepCopy() + + delete(node.Labels, labelName) + + return pnmr.client.Patch(ctx, &node, client.MergeFrom(nodeCopy)) +} diff --git a/controllers/pod_node_module_reconciler_test.go b/controllers/pod_node_module_reconciler_test.go new file mode 100644 index 000000000..d0c2075dd --- /dev/null +++ b/controllers/pod_node_module_reconciler_test.go @@ -0,0 +1,224 @@ +package controllers + +import ( + "context" + + "github.com/golang/mock/gomock" + mock_client "github.com/kubernetes-sigs/kernel-module-management/internal/client" + "github.com/kubernetes-sigs/kernel-module-management/internal/constants" + "github.com/kubernetes-sigs/kernel-module-management/internal/daemonset" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var _ = Describe("PodNodeModuleReconciler", func() { + Describe("Reconcile", func() { + const ( + moduleName = "module-name" + nodeName = "node-name" + podName = "pod-name" + podNamespace = "pod-namespace" + nodeLabel = "some node label" + ) + + var ( + kubeClient *mock_client.MockClient + r *PodNodeModuleReconciler + mockDC *daemonset.MockDaemonSetCreator + ) + + BeforeEach(func() { + ctrl := gomock.NewController(GinkgoT()) + kubeClient = mock_client.NewMockClient(ctrl) + mockDC = daemonset.NewMockDaemonSetCreator(ctrl) + r = NewPodNodeModuleReconciler(kubeClient, mockDC) + }) + + ctx := context.Background() + nn := types.NamespacedName{ + Namespace: podNamespace, + Name: podName, + } + req := ctrl.Request{NamespacedName: nn} + + It("should return an error if the pod is not labeled", func() { + gomock.InOrder( + kubeClient.EXPECT().Get(ctx, nn, gomock.AssignableToTypeOf(&v1.Pod{})), + ) + + _, err := r.Reconcile(ctx, req) + Expect(err).To(HaveOccurred()) + }) + + It("should unlabel the node when a Pod is not ready", func() { + pod := v1.Pod{} + podWithModuleName := v1.Pod{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{constants.ModuleNameLabel: moduleName}}, + Spec: v1.PodSpec{NodeName: nodeName}, + } + node := v1.Node{} + nodeWithEmptyLabels := v1.Node{ + ObjectMeta: metav1.ObjectMeta{Labels: make(map[string]string)}, + } + + gomock.InOrder( + kubeClient. + EXPECT(). + Get(ctx, nn, &pod). + Do(func(_ context.Context, _ types.NamespacedName, o client.Object) { + o.SetLabels(map[string]string{constants.ModuleNameLabel: moduleName}) + o.(*v1.Pod).Spec.NodeName = nodeName + }), + mockDC.EXPECT().GetNodeLabelFromPod(&podWithModuleName, moduleName).Return(nodeLabel), + kubeClient. + EXPECT(). + Get(ctx, types.NamespacedName{Name: nodeName}, &node). + Do(func(_ context.Context, _ types.NamespacedName, o client.Object) { + o.SetLabels(map[string]string{nodeLabel: ""}) + }), + kubeClient. + EXPECT(). + Patch(ctx, &nodeWithEmptyLabels, gomock.Any()). + Do(func(_ context.Context, n client.Object, p client.Patch, _ ...client.PatchOption) { + Expect(p.Type()).To(Equal(types.MergePatchType)) + Expect(p.Data(n)).To( + Equal( + []byte(`{"metadata":{"labels":null}}`), + ), + ) + }), + ) + + _, err := r.Reconcile(ctx, req) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should label the node when a Pod is ready", func() { + pod := v1.Pod{} + readyPod := v1.Pod{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{constants.ModuleNameLabel: moduleName}}, + Spec: v1.PodSpec{NodeName: nodeName}, + Status: v1.PodStatus{ + Conditions: []v1.PodCondition{ + { + Type: v1.PodReady, + Status: v1.ConditionTrue, + }, + }, + }, + } + node := v1.Node{} + nodeWithLabel := node + nodeWithLabel.SetLabels(map[string]string{nodeLabel: ""}) + patch := client.MergeFrom(&node) + + gomock.InOrder( + kubeClient. + EXPECT(). + Get(ctx, nn, &pod). + Do(func(_ context.Context, _ types.NamespacedName, o client.Object) { + o.SetLabels(map[string]string{constants.ModuleNameLabel: moduleName}) + o.(*v1.Pod).Spec.NodeName = nodeName + o.(*v1.Pod).Status.Conditions = []v1.PodCondition{ + { + Type: v1.PodReady, + Status: v1.ConditionTrue, + }, + } + }), + mockDC.EXPECT().GetNodeLabelFromPod(&readyPod, moduleName).Return(nodeLabel), + kubeClient.EXPECT().Get(ctx, types.NamespacedName{Name: nodeName}, &node), + kubeClient. + EXPECT(). + Patch(ctx, &nodeWithLabel, gomock.AssignableToTypeOf(patch)). + Do(func(_ context.Context, n client.Object, p client.Patch, _ ...client.PatchOption) { + Expect(p.Type()).To(Equal(types.MergePatchType)) + Expect(p.Data(n)).To( + Equal( + []byte(`{"metadata":{"labels":{"some node label":""}}}`), + ), + ) + }), + ) + + _, err := r.Reconcile(ctx, req) + Expect(err).NotTo(HaveOccurred()) + }) + + It("should unlabel the node and remove the pod finalizer when the pod is being deleted", func() { + now := metav1.Now() + + pod := v1.Pod{} + deletedPod := v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + DeletionTimestamp: &now, + Finalizers: []string{constants.NodeLabelerFinalizer}, + Labels: map[string]string{constants.ModuleNameLabel: moduleName}, + }, + Spec: v1.PodSpec{NodeName: nodeName}, + } + podWithoutFinalizer := v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + DeletionTimestamp: &now, + Finalizers: make([]string, 0), + Labels: map[string]string{constants.ModuleNameLabel: moduleName}, + }, + Spec: v1.PodSpec{NodeName: nodeName}, + } + + node := v1.Node{} + nodeWithEmptyLabels := v1.Node{ + ObjectMeta: metav1.ObjectMeta{Labels: make(map[string]string)}, + } + + gomock.InOrder( + kubeClient. + EXPECT(). + Get(ctx, nn, &pod). + Do(func(_ context.Context, _ types.NamespacedName, o client.Object) { + o.SetLabels(map[string]string{constants.ModuleNameLabel: moduleName}) + o.(*v1.Pod).Spec.NodeName = nodeName + o.SetDeletionTimestamp(&now) + o.SetFinalizers([]string{constants.NodeLabelerFinalizer}) + }), + mockDC.EXPECT().GetNodeLabelFromPod(&deletedPod, moduleName).Return(nodeLabel), + kubeClient. + EXPECT(). + Get(ctx, types.NamespacedName{Name: nodeName}, &node). + Do(func(_ context.Context, _ types.NamespacedName, o client.Object) { + o.SetLabels(map[string]string{nodeLabel: ""}) + }), + kubeClient. + EXPECT(). + Patch(ctx, &nodeWithEmptyLabels, gomock.Any()). + Do(func(_ context.Context, n client.Object, p client.Patch, _ ...client.PatchOption) { + Expect(p.Type()).To(Equal(types.MergePatchType)) + Expect(p.Data(n)).To( + Equal( + []byte(`{"metadata":{"labels":null}}`), + ), + ) + }), + kubeClient. + EXPECT(). + Patch(ctx, &podWithoutFinalizer, gomock.Any()). + Do(func(_ context.Context, po client.Object, p client.Patch, _ ...client.PatchOption) { + Expect(p.Type()).To(Equal(types.MergePatchType)) + Expect(p.Data(po)).To( + Equal( + []byte(`{"metadata":{"finalizers":null}}`), + ), + ) + }), + ) + + _, err := r.Reconcile(ctx, req) + Expect(err).NotTo(HaveOccurred()) + }) + }) +}) diff --git a/controllers/preflightvalidation_reconciler.go b/controllers/preflightvalidation_reconciler.go new file mode 100644 index 000000000..672fc5101 --- /dev/null +++ b/controllers/preflightvalidation_reconciler.go @@ -0,0 +1,206 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/source" + + kmmv1beta1 "github.com/kubernetes-sigs/kernel-module-management/api/v1beta1" + "github.com/kubernetes-sigs/kernel-module-management/internal/filter" + "github.com/kubernetes-sigs/kernel-module-management/internal/preflight" + "github.com/kubernetes-sigs/kernel-module-management/internal/statusupdater" + "github.com/kubernetes-sigs/kernel-module-management/internal/utils" +) + +const reconcileRequeueInSeconds = 60 + +// ClusterPreflightReconciler reconciles a PreflightValidation object +type PreflightValidationReconciler struct { + client client.Client + filter *filter.Filter + statusUpdater statusupdater.PreflightStatusUpdater + preflight preflight.PreflightAPI +} + +func NewPreflightValidationReconciler( + client client.Client, + filter *filter.Filter, + statusUpdater statusupdater.PreflightStatusUpdater, + preflight preflight.PreflightAPI) *PreflightValidationReconciler { + return &PreflightValidationReconciler{ + client: client, + filter: filter, + statusUpdater: statusUpdater, + preflight: preflight} +} + +func (r *PreflightValidationReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + Named("preflightvalidation"). + For(&kmmv1beta1.PreflightValidation{}). + Watches( + &source.Kind{Type: &kmmv1beta1.Module{}}, + handler.EnqueueRequestsFromMapFunc(r.filter.EnqueueAllPreflightValidations), + builder.WithPredicates(filter.PreflightReconcilerModulePredicate()), + ). + WithOptions(controller.Options{ + MaxConcurrentReconciles: 1, + }). + Complete(r) +} + +//+kubebuilder:rbac:groups=kmm.sigs.k8s.io,resources=modules,verbs=get;list;watch +//+kubebuilder:rbac:groups=kmm.sigs.k8s.io,resources=preflightvalidations,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=kmm.sigs.k8s.io,resources=preflightvalidations/status,verbs=get;update;patch + +// Reconcile Reconiliation entry point +func (r *PreflightValidationReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := ctrl.LoggerFrom(ctx) + log.Info("Start PreflightValidation Reconciliation") + + pv := kmmv1beta1.PreflightValidation{} + err := r.client.Get(ctx, req.NamespacedName, &pv) + if err != nil { + if apierrors.IsNotFound(err) { + log.Info("Reconciliation object not found; not reconciling") + return ctrl.Result{}, nil + } + log.Error(err, "preflight validation reconcile failed to find object") + return ctrl.Result{}, err + } + + reconCompleted, err := r.runPreflightValidation(ctx, &pv) + if err != nil { + log.Error(err, "runPreflightValidation failed") + return ctrl.Result{}, err + } + + if reconCompleted { + log.Info("PreflightValidation reconciliation success") + return ctrl.Result{}, nil + } + log.Info("PreflightValidation reconciliation requeue") + return ctrl.Result{RequeueAfter: time.Second * reconcileRequeueInSeconds}, nil +} + +func (r *PreflightValidationReconciler) runPreflightValidation(ctx context.Context, pv *kmmv1beta1.PreflightValidation) (bool, error) { + log := ctrl.LoggerFrom(ctx) + + modulesToCheck, err := r.getModulesToCheck(ctx, pv) + if err != nil { + return false, fmt.Errorf("failed to get list of modules to check for preflight: %w", err) + } + + for _, module := range modulesToCheck { + log.Info("start module preflight validation", "name", module.Name) + + verified, message := r.preflight.PreflightUpgradeCheck(ctx, &module, pv.Spec.KernelVersion) + + log.Info("module preflight validation result", "name", module.Name, "verified", verified) + + r.updatePreflightStatus(ctx, pv, module.Name, message, verified) + } + + return r.checkPreflightCompletion(ctx, pv.Name, pv.Namespace) +} + +func (r *PreflightValidationReconciler) getModulesToCheck(ctx context.Context, pv *kmmv1beta1.PreflightValidation) ([]kmmv1beta1.Module, error) { + log := ctrl.LoggerFrom(ctx) + + modulesList := kmmv1beta1.ModuleList{} + err := r.client.List(ctx, &modulesList) + if err != nil { + return nil, fmt.Errorf("failed to get list of all Modules: %w", err) + } + + err = r.presetModulesStatuses(ctx, pv, modulesList.Items) + if err != nil { + return nil, fmt.Errorf("failed to preset new modules' statuses: %w", err) + } + + modulesToCheck := make([]kmmv1beta1.Module, 0, len(modulesList.Items)) + for _, module := range modulesList.Items { + if module.GetDeletionTimestamp() != nil { + log.Info("Module is marked for deletion, skipping preflight validation", "name", module.Name) + continue + } + if pv.Status.CRStatuses[module.Name].VerificationStatus != kmmv1beta1.VerificationTrue { + modulesToCheck = append(modulesToCheck, module) + } + } + return modulesToCheck, nil +} + +func (r *PreflightValidationReconciler) updatePreflightStatus(ctx context.Context, pv *kmmv1beta1.PreflightValidation, moduleName, message string, verified bool) { + log := ctrl.LoggerFrom(ctx) + verificationStatus := kmmv1beta1.VerificationFalse + if verified { + verificationStatus = kmmv1beta1.VerificationTrue + } + err := r.statusUpdater.PreflightSetVerificationStatus(ctx, pv, moduleName, verificationStatus, message) + if err != nil { + log.Info(utils.WarnString("failed to update the status of Module CR in preflight"), "module", moduleName, "error", err) + } +} + +func (r *PreflightValidationReconciler) presetModulesStatuses(ctx context.Context, pv *kmmv1beta1.PreflightValidation, modules []kmmv1beta1.Module) error { + if pv.Status.CRStatuses == nil { + pv.Status.CRStatuses = make(map[string]*kmmv1beta1.CRStatus, len(modules)) + } + existingModulesName := sets.NewString() + newModulesNames := make([]string, 0, len(modules)) + for _, module := range modules { + if module.GetDeletionTimestamp() != nil { + continue + } + existingModulesName.Insert(module.Name) + if _, ok := pv.Status.CRStatuses[module.Name]; ok { + continue + } + newModulesNames = append(newModulesNames, module.Name) + } + return r.statusUpdater.PreflightPresetStatuses(ctx, pv, existingModulesName, newModulesNames) +} + +func (r *PreflightValidationReconciler) checkPreflightCompletion(ctx context.Context, name, namespace string) (bool, error) { + pv := kmmv1beta1.PreflightValidation{} + err := r.client.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, &pv) + if err != nil { + return false, fmt.Errorf("failed to get preflight validation object in checkPreflightCompletion: %w", err) + } + + for modName, crStatus := range pv.Status.CRStatuses { + if crStatus.VerificationStatus != kmmv1beta1.VerificationTrue { + ctrl.LoggerFrom(ctx).Info("at least one Module is not verified yet", "module", modName, "status", crStatus.VerificationStatus) + return false, nil + } + } + + return true, nil +} diff --git a/controllers/preflightvalidation_reconciler_test.go b/controllers/preflightvalidation_reconciler_test.go new file mode 100644 index 000000000..95590f348 --- /dev/null +++ b/controllers/preflightvalidation_reconciler_test.go @@ -0,0 +1,330 @@ +package controllers + +import ( + "context" + "fmt" + "time" + + "github.com/golang/mock/gomock" + kmmv1beta1 "github.com/kubernetes-sigs/kernel-module-management/api/v1beta1" + "github.com/kubernetes-sigs/kernel-module-management/internal/client" + "github.com/kubernetes-sigs/kernel-module-management/internal/preflight" + "github.com/kubernetes-sigs/kernel-module-management/internal/statusupdater" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +const ( + preflightName = "test-preflight" +) + +var _ = Describe("Reconcile", func() { + var ( + ctrl *gomock.Controller + clnt *client.MockClient + mockSU *statusupdater.MockPreflightStatusUpdater + mockPreflight *preflight.MockPreflightAPI + req reconcile.Request + ctx context.Context + nsn types.NamespacedName + pr *PreflightValidationReconciler + ) + + BeforeEach(func() { + ctrl = gomock.NewController(GinkgoT()) + clnt = client.NewMockClient(ctrl) + mockSU = statusupdater.NewMockPreflightStatusUpdater(ctrl) + mockPreflight = preflight.NewMockPreflightAPI(ctrl) + nsn = types.NamespacedName{ + Name: preflightName, + Namespace: namespace, + } + req = reconcile.Request{NamespacedName: nsn} + ctx = context.Background() + pr = NewPreflightValidationReconciler(clnt, nil, mockSU, mockPreflight) + }) + + It("should do nothing if the Preflight is not available anymore", func() { + clnt.EXPECT().Get(ctx, nsn, &kmmv1beta1.PreflightValidation{}).Return(apierrors.NewNotFound(schema.GroupResource{}, preflightName)) + + res, err := pr.Reconcile(ctx, req) + + Expect(err).To(BeNil()) + Expect(res).To(Equal(reconcile.Result{})) + }) + + It("should return error when failed to get preflight", func() { + clnt.EXPECT().Get(ctx, nsn, &kmmv1beta1.PreflightValidation{}).Return(fmt.Errorf("some error")) + + res, err := pr.Reconcile(ctx, req) + + Expect(err).To(HaveOccurred()) + Expect(res).To(Equal(reconcile.Result{})) + }) + + It("good flow, all verified", func() { + mod := kmmv1beta1.Module{ + ObjectMeta: metav1.ObjectMeta{ + Name: "moduleName", + }, + } + pv := kmmv1beta1.PreflightValidation{ + ObjectMeta: metav1.ObjectMeta{ + Name: nsn.Name, + Namespace: nsn.Namespace, + }, + Spec: kmmv1beta1.PreflightValidationSpec{ + KernelVersion: "some kernel version", + }, + Status: kmmv1beta1.PreflightValidationStatus{ + CRStatuses: map[string]*kmmv1beta1.CRStatus{mod.Name: &kmmv1beta1.CRStatus{}}, + }, + } + gomock.InOrder( + clnt.EXPECT().Get(context.Background(), nsn, &kmmv1beta1.PreflightValidation{}).DoAndReturn( + func(_ interface{}, _ interface{}, m *kmmv1beta1.PreflightValidation) error { + m.ObjectMeta = pv.ObjectMeta + m.Spec.KernelVersion = pv.Spec.KernelVersion + m.Status = pv.Status + return nil + }, + ), + clnt.EXPECT().List(ctx, gomock.Any(), gomock.Any()).DoAndReturn( + func(_ interface{}, list *kmmv1beta1.ModuleList, _ ...interface{}) error { + list.Items = []kmmv1beta1.Module{mod} + return nil + }, + ), + mockSU.EXPECT().PreflightPresetStatuses(ctx, &pv, sets.NewString(mod.Name), []string{}).Return(nil), + mockPreflight.EXPECT().PreflightUpgradeCheck(ctx, &mod, "some kernel version").Return(true, "some message"), + mockSU.EXPECT().PreflightSetVerificationStatus(ctx, &pv, mod.Name, kmmv1beta1.VerificationTrue, "some message").Return(nil), + clnt.EXPECT().Get(context.Background(), nsn, &kmmv1beta1.PreflightValidation{}).DoAndReturn( + func(_ interface{}, _ interface{}, m *kmmv1beta1.PreflightValidation) error { + m.Status.CRStatuses = map[string]*kmmv1beta1.CRStatus{mod.Name: &kmmv1beta1.CRStatus{VerificationStatus: "True"}} + return nil + }, + ), + ) + + res, err := pr.Reconcile(ctx, req) + + Expect(err).To(BeNil()) + Expect(res).To(Equal(reconcile.Result{})) + }) + + It("good flow, some not verified", func() { + mod := kmmv1beta1.Module{ + ObjectMeta: metav1.ObjectMeta{ + Name: "moduleName", + }, + } + pv := kmmv1beta1.PreflightValidation{ + ObjectMeta: metav1.ObjectMeta{ + Name: nsn.Name, + Namespace: nsn.Namespace, + }, + Spec: kmmv1beta1.PreflightValidationSpec{ + KernelVersion: "some kernel version", + }, + Status: kmmv1beta1.PreflightValidationStatus{ + CRStatuses: map[string]*kmmv1beta1.CRStatus{mod.Name: &kmmv1beta1.CRStatus{}}, + }, + } + gomock.InOrder( + clnt.EXPECT().Get(context.Background(), nsn, &kmmv1beta1.PreflightValidation{}).DoAndReturn( + func(_ interface{}, _ interface{}, m *kmmv1beta1.PreflightValidation) error { + m.ObjectMeta = pv.ObjectMeta + m.Spec.KernelVersion = pv.Spec.KernelVersion + m.Status = pv.Status + return nil + }, + ), + clnt.EXPECT().List(ctx, gomock.Any(), gomock.Any()).DoAndReturn( + func(_ interface{}, list *kmmv1beta1.ModuleList, _ ...interface{}) error { + list.Items = []kmmv1beta1.Module{mod} + return nil + }, + ), + mockSU.EXPECT().PreflightPresetStatuses(ctx, &pv, sets.NewString(mod.Name), []string{}).Return(nil), + mockPreflight.EXPECT().PreflightUpgradeCheck(ctx, &mod, "some kernel version").Return(true, "some message"), + mockSU.EXPECT().PreflightSetVerificationStatus(ctx, &pv, mod.Name, kmmv1beta1.VerificationTrue, "some message").Return(nil), + clnt.EXPECT().Get(context.Background(), nsn, &kmmv1beta1.PreflightValidation{}).DoAndReturn( + func(_ interface{}, _ interface{}, m *kmmv1beta1.PreflightValidation) error { + m.Status.CRStatuses = map[string]*kmmv1beta1.CRStatus{mod.Name: &kmmv1beta1.CRStatus{VerificationStatus: "False"}} + return nil + }, + ), + ) + + res, err := pr.Reconcile(ctx, req) + + Expect(err).To(BeNil()) + Expect(res).To(Equal(reconcile.Result{RequeueAfter: time.Second * reconcileRequeueInSeconds})) + }) + +}) + +var _ = Describe("getModulesCheck", func() { + var ( + ctrl *gomock.Controller + clnt *client.MockClient + mockSU *statusupdater.MockPreflightStatusUpdater + mockPreflight *preflight.MockPreflightAPI + ctx context.Context + pr *PreflightValidationReconciler + ) + + BeforeEach(func() { + ctrl = gomock.NewController(GinkgoT()) + clnt = client.NewMockClient(ctrl) + mockSU = statusupdater.NewMockPreflightStatusUpdater(ctrl) + mockPreflight = preflight.NewMockPreflightAPI(ctrl) + ctx = context.Background() + pr = NewPreflightValidationReconciler(clnt, nil, mockSU, mockPreflight) + }) + + It("multiple modules, statuses exist, none deleted", func() { + pv := kmmv1beta1.PreflightValidation{ + ObjectMeta: metav1.ObjectMeta{ + Name: preflightName, + Namespace: namespace, + }, + Spec: kmmv1beta1.PreflightValidationSpec{ + KernelVersion: "some kernel version", + }, + Status: kmmv1beta1.PreflightValidationStatus{ + CRStatuses: map[string]*kmmv1beta1.CRStatus{ + "moduleName1": &kmmv1beta1.CRStatus{}, + "moduleName2": &kmmv1beta1.CRStatus{}}, + }, + } + mod1 := kmmv1beta1.Module{ + ObjectMeta: metav1.ObjectMeta{ + Name: "moduleName1", + }, + } + + mod2 := kmmv1beta1.Module{ + ObjectMeta: metav1.ObjectMeta{ + Name: "moduleName2", + }, + } + + clnt.EXPECT().List(ctx, gomock.Any(), gomock.Any()).DoAndReturn( + func(_ interface{}, list *kmmv1beta1.ModuleList, _ ...interface{}) error { + list.Items = []kmmv1beta1.Module{mod1, mod2} + return nil + }, + ) + mockSU.EXPECT().PreflightPresetStatuses(ctx, &pv, sets.NewString("moduleName1", "moduleName2"), []string{}) + + modulesToCheck, err := pr.getModulesToCheck(ctx, &pv) + + Expect(err).To(BeNil()) + Expect(modulesToCheck).To(Equal([]kmmv1beta1.Module{mod1, mod2})) + + }) + + It("multiple modules, one status missing, none deleted", func() { + pv := kmmv1beta1.PreflightValidation{ + ObjectMeta: metav1.ObjectMeta{ + Name: preflightName, + Namespace: namespace, + }, + Spec: kmmv1beta1.PreflightValidationSpec{ + KernelVersion: "some kernel version", + }, + Status: kmmv1beta1.PreflightValidationStatus{ + CRStatuses: map[string]*kmmv1beta1.CRStatus{"moduleName1": &kmmv1beta1.CRStatus{}}, + }, + } + mod1 := kmmv1beta1.Module{ + ObjectMeta: metav1.ObjectMeta{ + Name: "moduleName1", + }, + } + + mod2 := kmmv1beta1.Module{ + ObjectMeta: metav1.ObjectMeta{ + Name: "moduleName2", + }, + } + gomock.InOrder( + clnt.EXPECT().List(ctx, gomock.Any(), gomock.Any()).DoAndReturn( + func(_ interface{}, list *kmmv1beta1.ModuleList, _ ...interface{}) error { + list.Items = []kmmv1beta1.Module{mod1, mod2} + return nil + }, + ), + mockSU.EXPECT().PreflightPresetStatuses(ctx, &pv, sets.NewString("moduleName1", "moduleName2"), []string{"moduleName2"}).DoAndReturn( + func(_ interface{}, pv *kmmv1beta1.PreflightValidation, existingModules sets.String, newModules []string) error { + pv.Status.CRStatuses[newModules[0]] = &kmmv1beta1.CRStatus{} + return nil + }), + ) + + modulesToCheck, err := pr.getModulesToCheck(ctx, &pv) + + Expect(err).To(BeNil()) + Expect(modulesToCheck).To(Equal([]kmmv1beta1.Module{mod1, mod2})) + }) + + It("multiple modules, one status missing, one deleted", func() { + pv := kmmv1beta1.PreflightValidation{ + ObjectMeta: metav1.ObjectMeta{ + Name: preflightName, + Namespace: namespace, + }, + Spec: kmmv1beta1.PreflightValidationSpec{ + KernelVersion: "some kernel version", + }, + Status: kmmv1beta1.PreflightValidationStatus{ + CRStatuses: map[string]*kmmv1beta1.CRStatus{"moduleName1": &kmmv1beta1.CRStatus{}}, + }, + } + mod1 := kmmv1beta1.Module{ + ObjectMeta: metav1.ObjectMeta{ + Name: "moduleName1", + }, + } + + mod2 := kmmv1beta1.Module{ + ObjectMeta: metav1.ObjectMeta{ + Name: "moduleName2", + }, + } + + mod3 := kmmv1beta1.Module{ + ObjectMeta: metav1.ObjectMeta{ + Name: "moduleName3", + }, + } + timestamp := metav1.Now() + mod3.SetDeletionTimestamp(×tamp) + + gomock.InOrder( + clnt.EXPECT().List(ctx, gomock.Any(), gomock.Any()).DoAndReturn( + func(_ interface{}, list *kmmv1beta1.ModuleList, _ ...interface{}) error { + list.Items = []kmmv1beta1.Module{mod1, mod2, mod3} + return nil + }, + ), + mockSU.EXPECT().PreflightPresetStatuses(ctx, &pv, sets.NewString("moduleName1", "moduleName2"), []string{"moduleName2"}).DoAndReturn( + func(_ interface{}, pv *kmmv1beta1.PreflightValidation, existingModules sets.String, newModules []string) error { + pv.Status.CRStatuses[newModules[0]] = &kmmv1beta1.CRStatus{} + delete(pv.Status.CRStatuses, "moduleName3") + return nil + }), + ) + modulesToCheck, err := pr.getModulesToCheck(ctx, &pv) + + Expect(err).To(BeNil()) + Expect(modulesToCheck).To(Equal([]kmmv1beta1.Module{mod1, mod2})) + }) +}) diff --git a/controllers/suite_test.go b/controllers/suite_test.go new file mode 100644 index 000000000..81befa57a --- /dev/null +++ b/controllers/suite_test.go @@ -0,0 +1,40 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "testing" + + "github.com/kubernetes-sigs/kernel-module-management/internal/test" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/runtime" + //+kubebuilder:scaffold:imports +) + +var scheme *runtime.Scheme + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + + var err error + + scheme, err = test.TestScheme() + Expect(err).NotTo(HaveOccurred()) + + RunSpecs(t, "Controller Suite") +} diff --git a/docs.mk b/docs.mk new file mode 100644 index 000000000..9240c6c27 --- /dev/null +++ b/docs.mk @@ -0,0 +1,8 @@ +.PHONY: docs + +docs: bin/plantuml.jar + make -C docs + +bin/plantuml.jar: + mkdir -p bin + curl -Lo $@ https://sourceforge.net/projects/plantuml/files/1.2022.2/plantuml.1.2022.2.jar/download \ No newline at end of file diff --git a/docs/.nojekyll b/docs/.nojekyll new file mode 100644 index 000000000..e69de29bb diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 000000000..dc6beb923 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,13 @@ +all: diagrams build + +.PHONY: diagrams + +diagrams: + cd mkdocs/developer && $(MAKE) + +build: + mkdocs build + +clean: + cd mkdocs/developer && $(MAKE) clean + rm -fr site diff --git a/docs/index.html b/docs/index.html new file mode 100644 index 000000000..1732d5fda --- /dev/null +++ b/docs/index.html @@ -0,0 +1,7 @@ + + + +OOT Operator docs + + + diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml new file mode 100644 index 000000000..e12512061 --- /dev/null +++ b/docs/mkdocs.yml @@ -0,0 +1,12 @@ +site_name: Out-Of-Tree operator +repo_url: https://github.com/kubernetes-sigs/kernel-module-management +edit_uri: edit/main/docs/ + +docs_dir: mkdocs +strict: true + +plugins: + - exclude: + glob: + - developer/diagrams/*.puml + - developer/Makefile diff --git a/docs/mkdocs/developer/Makefile b/docs/mkdocs/developer/Makefile new file mode 100644 index 000000000..35bd20876 --- /dev/null +++ b/docs/mkdocs/developer/Makefile @@ -0,0 +1,7 @@ +all: diagrams/reconciliation-module.png + +diagrams/%.png: diagrams/%.puml + java -jar ../../../bin/plantuml.jar -nometadata -p < $< > $@ + +clean: + rm -f *.png diff --git a/docs/mkdocs/developer/diagrams/.gitignore b/docs/mkdocs/developer/diagrams/.gitignore new file mode 100644 index 000000000..e33609d25 --- /dev/null +++ b/docs/mkdocs/developer/diagrams/.gitignore @@ -0,0 +1 @@ +*.png diff --git a/docs/mkdocs/developer/diagrams/reconciliation-module.puml b/docs/mkdocs/developer/diagrams/reconciliation-module.puml new file mode 100644 index 000000000..d34c4f937 --- /dev/null +++ b/docs/mkdocs/developer/diagrams/reconciliation-module.puml @@ -0,0 +1,80 @@ +@startuml +'https://plantuml.com/activity-diagram-beta + +skinparam ConditionEndStyle hline + +title ""Module"" Reconciliation + +'start + +split + -[hidden]-> + :A ""Module"" changes; +split again + -[hidden]-> + :A managed ""DaemonSet"" changes; +split again + -[hidden]-> + :A managed build object changes; +split again + -[hidden]-> + :a ""Node"" tagged with its kernel version + has one of its labels changed; +end split + +:List all nodes that match the ""Module""'s "".spec.selector""; + +partition "Create mappings" { + while (there are unprocessed nodes) is (next node) + note + We pick the first mapping in the ""Module""'s + "".spec.kernelMappings"" that matches the node's + kernel (either literal or regexp). + end note + + if (do we have a mapping for this kernel?) then (yes) + :Add the mapping to ""mappings""; + else (no) + endif + endwhile (done) +} + +:List all existing DaemonSets managed by this module; + +partition "Sync DaemonSets" { + while (there are unprocessed ""mappings"") is (next mapping); + partition "In-cluster build" { + if (is in-cluster build configured for this mapping?) then (no) + (yes) elseif (does the image already exist?) then (no) + if (are we already building that image?) then (no) + :Create the build; + else (yes) + endif + + :Requeue a reconciliation + for this ""Module""; + endif + } + + :Generate a DaemonSet spec for this mapping; + :Call ""CreateOrPatch()"" for this DaemonSet; + endwhile (done) +} + +partition "Garbage collect obsolete DaemonSets" { + note + This deletes DaemonSets targeting a kernel + that no nodes are running anymore. + end note + + while (there are unprocessed existing ""DaemonSets"") is (next ""DaemonSet""); + if (is this kernel in ""mappings""?) then (no) + :Delete the DaemonSet; + else (yes) + endif + endwhile (done) +} + +stop + +@enduml diff --git a/docs/mkdocs/developer/index.md b/docs/mkdocs/developer/index.md new file mode 100644 index 000000000..d51f3f4d5 --- /dev/null +++ b/docs/mkdocs/developer/index.md @@ -0,0 +1,102 @@ +# Design notes + +Kernel Module Management brings a new `Module` CRD. +`Module` represents an out-of-tree kernel module that must be inserted into the cluster nodes’ kernel through a +DriverContainer scheduled by a `DaemonSet`. + +KMM optionally builds and runs DriverContainers on the cluster. +It picks the right DriverContainer image by leveraging kernel mappings in the `Module` CRD that describe which image +should be used for which kernel. + +A kernel mapping maps either a literal kernel name or a regex with a kernel’s node. +This allows for more flexibility when targeting a set of kernels (e.g. “for Ubuntu nodes, build from that repository”). +Variables available at build time still reflect the actual kernel version. + +## `Module` CRD +```yaml +apiVersion: ooto.sigs.k8s.io/v1alpha1 +kind: Module +metadata: + name: module-sample +spec: + additionalVolumes: + # All volumes here are available into the DriverContainer / DevicePlugin pod. + # Users should mount those by adding corresponding volumeMounts under .spec.devicePlugin + # and .spec.moduleLoader. + - name: some-volume + configMap: + name: some-configmap + devicePlugin: # is a Container spec + # This container will be privileged and will mount + # /var/lib/kubelet/device-plugins automatically. + name: overwritten-anyway + image: some-image + moduleLoader: # is a Container spec + # This container will not be privileged by default. + # It will mount /lib/modules and /usr/lib/modules automatically. + name: overwritten-anyway + image: overwritten-anyway + securityContext: + capabilities: + add: [SYS_MODULE] # this is enough in most cases + kernelMappings: + - literal: 5.16.11-200.fc35.x86_64 + containerImage: quay.io/vendor/module-sample:fedora-5.16.11-200.fc35.x86_64 + - literal: 5.4.0-1054-gke + containerImage: quay.io/vendor/module-sample:ubuntu-5.4.0-1054-gke + - regexp: '^.*\-gke$' + build: + buildArgs: + - name: SOME_KEY + value: SOME_VALUE + pull: + insecure: false + push: + insecure: false + dockerfile: | + FROM some-image + RUN some-command + containerImage: quay.io/vendor/module-sample:gke + selector: # top-level selector + feature.node.kubernetes.io/cpu-cpuid.VMX: true +status: + conditions: + - lastTransitionTime: "2022-03-07T14:01:00Z" + status: False + type: Progressing + - lastTransitionTime: "2022-03-07T14:01:01Z" + status: True + type: Ready +``` + +## In-cluster builds +Virtually any Kubernetes distribution ships with its own kernel. +It is thus challenging for a kernel module vendor to make DriverContainer images available for all kernels available +in all Kubernetes distributions out there (or even for a subset of them). + +KMM supports in-cluster builds of DriverContainer images when those are not made available by the vendor. +The in-cluster build system is able to build an image from any Git repository that contains a `Dockerfile`. +It can then push said image to a user-provided repository, optionally using authentication provided in a Kubernetes secret. + +**Optional:** on upstream Kubernetes, we may want to deploy an in-cluster registry to host in-cluster built images. +**On OCP**, the build mechanism would be BuildConfig (maybe Shipwright in the future) and we can leverage the +integrated in-cluster registry. + +## Unloading modules +Users must provide a [preStop lifecycle hook](https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/) +to their DriverContainer pod template to make sure that their module is unloaded when the DriverContainer pod exits. + +## Security + +### DriverContainer privileges +By default, the operator would only grant the [`CAP_SYS_MODULE`](https://man7.org/linux/man-pages/man7/capabilities.7.html) +capability to DriverContainer `DaemonSets`. + +### Kubernetes API privileges +The KMM Operator would only be granted a limited set of Kubernetes API privileges: + +- Read, modify (for kernel version labeling) and watch `Nodes`; +- Read and watch `Modules`, update their status; +- Read, create, modify and watch `DaemonSets`; +- Read, create, modify and watch `Build` objects (from whatever system we agree on); +- Read `Secrets` for pull and build secrets. diff --git a/docs/mkdocs/developer/reconciliation_loops.md b/docs/mkdocs/developer/reconciliation_loops.md new file mode 100644 index 000000000..03865bde9 --- /dev/null +++ b/docs/mkdocs/developer/reconciliation_loops.md @@ -0,0 +1,20 @@ +# Reconciliation loop + +## Modules + +Each time a new `Module` is created, we need to find to which nodes it applies. +A first filtering is performed using the `.spec.selector` field, then we go through the module’s kernel mappings to +find a container image that matches the node’s kernel. +We end up with a certain number of (kernel, image) pairs; for each of these pairs, there should be a `DaemonSet`. +We first look for a `DaemonSet` that would already be targeting the same kernel and DriverContainer image (that data is +stored in the `DaemonSet`’s labels). +If there is already such a `DaemonSet`, we patch it, if needed. +If there is not already a matching `DaemonSet`, we create it and set the `Module` as owner. + +When a `Module` is deleted, we have nothing to do: because we set it as owner of all `DaemonSets`, Kubernetes garbage +collection will take care of deleting them. + +We watch `Module`, owned `DaemonSet` and build objects as well as nodes to make sure that we are not missing any change +in the cluster. + +![Modules reconciliation](diagrams/reconciliation-module.png) \ No newline at end of file diff --git a/docs/mkdocs/developer/use_cases.md b/docs/mkdocs/developer/use_cases.md new file mode 100644 index 000000000..b35fd0577 --- /dev/null +++ b/docs/mkdocs/developer/use_cases.md @@ -0,0 +1,46 @@ +# Use cases + +## A new module is added to the cluster +The operator lists all nodes required to build the module and looks at their kernels. +It uses the `Module`’s `.spec.kernelMappings` to create `DaemonSets` for each of those kernels. + +If an image for one of these `DaemonSets` requires an in-cluster build, it creates the build object (build system TBD). +The `Module` owns the build object, so that it is deleted when the `Module` is deleted. + +## Module update: kernel `$kernel` should run a new DriverContainer image +The OOTO would be triggered by an update to any `Module` instance. +That reconciliation generates a list of `DaemonSets` that need to be reconciled. + +For each `DaemonSet` belonging to `Module`, we set the following labels: + +| key | value | +|----------------------------------------------|---------------| +| `ooto.sigs.k8s.io/module-name` | `$moduleName` | +| `oot.node.kubernetes.io/kernel-version.full` | `$kernel` | + +We either create those `DaemonSets` (if they do not already exist) or update them (if a `DaemonSet` already exists with +the same labels). +If a `DaemonSet` already exists for `$moduleName` and `$kernel`, it is updated with the new DriverContainer image. +If that image requires a build, we create the build object and wait for its completion before we modify the `DaemonSet`. + +## A new node joins the cluster (or is modified) +The OOTO lists all `Modules` and checks which ones should run on the node. +For each `Module` that should run on the node (determined by the `.spec.selector` field), the operator tries to find +a suitable DriverContainer image for the node’s kernel. +It creates a `DaemonSet` that runs that DriverContainer on all nodes running that kernel. + +## A node leaves the cluster +If that node was targeted by one of the `DaemonSets`, said `DaemonSet`’s `.status.desiredNumberScheduled` is +decremented. +The operator watches changes to those objects; if that field reaches 0, the `DaemonSet` is deleted. + +## A node’s kernel is updated +Depending on the Kubernetes distribution used, this is functionally equivalent to either: + +- Deleting the node and creating a new one, or; +- Modifying an existing node. + +That use case is thus covered above. + +## A node matches a Module’s .spec.selector, but not any of its kernel mappings +No DaemonSet is scheduled for this module and this kernel. The OOT kernel module will not be deployed on that node. diff --git a/docs/mkdocs/index.md b/docs/mkdocs/index.md new file mode 100644 index 000000000..6fea4c540 --- /dev/null +++ b/docs/mkdocs/index.md @@ -0,0 +1,10 @@ +# Home + +Kernel Module Management optionally builds and runs DriverContainers on Kubernetes clusters. + +## Getting started + +Install the bleeding edge version of the KMM Operator by running the following command: +```shell +kubectl apply -k https://github.com/kubernetes-sigs/kernel-module-management/config/default +``` diff --git a/docs/mkdocs/module_loaders.md b/docs/mkdocs/module_loaders.md new file mode 100644 index 000000000..bfbeeed81 --- /dev/null +++ b/docs/mkdocs/module_loaders.md @@ -0,0 +1,12 @@ +# How to build module-loaders to be used by KMMO + +### Pre-build module-loaders + +When a `Module` CR is applied to the cluster, KMMO will create a DaemonSet to run the module-loader specified in it. + +In order for module-loaders that depend on in-tree kernel-modules to work, KMMO is mounting `/lib/modules/${KVER}` from the node to the container, which means that it will override all files located in `/lib/modules/${KVER}` inside the module-loader. + +Therefore, when building a module-loader that is going to be used by KMMO, there are few things that needs to be done in the module-loader: +* Put the `*.ko` files in `/opt/lib/modules/${KVER}` instead of `/lib/modules/${KVER}` +* Link `/lib/modules/${KVER}` inside `/opt/lib/modules/$(KVER)/system` in case the module-loader depend on in-tree kernel-modules +* Run `depmod -b /opt` in order to generate the dependency file correctly diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 000000000..94b17df3c --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,2 @@ +mkdocs==1.3.0 +mkdocs-exclude==1.0.2 diff --git a/docs/site/404.html b/docs/site/404.html new file mode 100644 index 000000000..92eee0c25 --- /dev/null +++ b/docs/site/404.html @@ -0,0 +1,136 @@ + + + + + + + + + + + Out-Of-Tree operator + + + + + + + + + + + + +
+
+ Out-Of-Tree operator + + + + + +
+
+ +
+
+ +
+
+

404

+

Page not found

+
+
+ + +
+
+ +
+
+

Documentation built with MkDocs.

+
+ + + + + + + diff --git a/docs/site/css/base.css b/docs/site/css/base.css new file mode 100644 index 000000000..13f6c9188 --- /dev/null +++ b/docs/site/css/base.css @@ -0,0 +1,318 @@ +html { + /* csslint ignore:start */ + /* The nav header is 3.5rem high, plus 20px for the margin-top of the + main container. */ + scroll-padding-top: calc(3.5rem + 20px); + /* csslint ignore:end */ +} + +/* Replacement for `body { background-attachment: fixed; }`, which has + performance issues when scrolling on large displays. See #1394. */ +body::before { + content: ' '; + position: fixed; + width: 100%; + height: 100%; + top: 0; + left: 0; + background-color: #f8f8f8; + background: url(../img/grid.png) repeat-x; + will-change: transform; + z-index: -1; +} + +body > .container { + margin-top: 20px; + min-height: 400px; +} + +.navbar.fixed-top { /* csslint allow: adjoining-classes */ + /* csslint ignore:start */ + position: -webkit-sticky; + position: sticky; + /* csslint ignore:end */ +} + +.source-links { + float: right; +} + +.col-md-9 img { + max-width: 100%; + display: inline-block; + padding: 4px; + line-height: 1.428571429; + background-color: #fff; + border: 1px solid #ddd; + border-radius: 4px; + margin: 20px auto 30px auto; +} + +h1 { + color: #444; + font-weight: 400; + font-size: 42px; +} + +h2, h3, h4, h5, h6 { + color: #444; + font-weight: 300; +} + +hr { + border-top: 1px solid #aaa; +} + +pre, .rst-content tt { + max-width: 100%; + background: #fff; + border: solid 1px #e1e4e5; + color: #333; + overflow-x: auto; +} + +code.code-large, .rst-content tt.code-large { + font-size: 90%; +} + +code { + padding: 2px 5px; + background: #fff; + border: solid 1px #e1e4e5; + color: #333; + white-space: pre-wrap; + word-wrap: break-word; +} + +pre code { + display: block; + background: transparent; + border: none; + white-space: pre; + word-wrap: normal; + font-family: SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace; + font-size: 12px; +} + +kbd { + padding: 2px 4px; + font-size: 90%; + color: #fff; + background-color: #333; + border-radius: 3px; + -webkit-box-shadow: inset 0 -1px 0 rgba(0,0,0,.25); + box-shadow: inset 0 -1px 0 rgba(0,0,0,.25); +} + +a code { + color: #2FA4E7; +} + +a:hover code, a:focus code { + color: #157AB5; +} + +footer { + margin-top: 30px; + margin-bottom: 10px; + text-align: center; + font-weight: 200; +} + +.modal-dialog { + margin-top: 60px; +} + +/* + * Side navigation + * + * Scrollspy and affixed enhanced navigation to highlight sections and secondary + * sections of docs content. + */ + +.bs-sidebar.affix { /* csslint allow: adjoining-classes */ + /* csslint ignore:start */ + position: -webkit-sticky; + position: sticky; + /* csslint ignore:end */ + /* The nav header is 3.5rem high, plus 20px for the margin-top of the + main container. */ + top: calc(3.5rem + 20px); +} + +.bs-sidebar.card { /* csslint allow: adjoining-classes */ + padding: 0; + max-height: 90%; + overflow-y: auto; +} + +/* Toggle (vertically flip) sidebar collapse icon */ +.bs-sidebar .navbar-toggler span { + -moz-transform: scale(1, -1); + -webkit-transform: scale(1, -1); + -o-transform: scale(1, -1); + -ms-transform: scale(1, -1); + transform: scale(1, -1); +} + +.bs-sidebar .navbar-toggler.collapsed span { /* csslint allow: adjoining-classes */ + -moz-transform: scale(1, 1); + -webkit-transform: scale(1, 1); + -o-transform: scale(1, 1); + -ms-transform: scale(1, 1); + transform: scale(1, 1); +} + +/* First level of nav */ +.bs-sidebar > .navbar-collapse > .nav { + padding-top: 10px; + padding-bottom: 10px; + border-radius: 5px; + width: 100%; +} + +/* All levels of nav */ +.bs-sidebar .nav > li > a { + display: block; + padding: 5px 20px; + z-index: 1; +} +.bs-sidebar .nav > li > a:hover, +.bs-sidebar .nav > li > a:focus { + text-decoration: none; + border-right: 1px solid; +} +.bs-sidebar .nav > li > a.active, +.bs-sidebar .nav > li > a.active:hover, +.bs-sidebar .nav > li > a.active:focus { + font-weight: bold; + background-color: transparent; + border-right: 1px solid; +} + +.bs-sidebar .nav .nav .nav { + margin-left: 1em; +} + +.bs-sidebar .nav > li > a { + font-weight: bold; +} + +.bs-sidebar .nav .nav > li > a { + font-weight: normal; +} + +.headerlink { + font-family: FontAwesome; + font-size: 14px; + display: none; + padding-left: .5em; +} + +h1:hover .headerlink, h2:hover .headerlink, h3:hover .headerlink, h4:hover .headerlink, h5:hover .headerlink, h6:hover .headerlink{ + display:inline-block; +} + + + +.admonition { + padding: 15px; + margin-bottom: 20px; + border: 1px solid transparent; + border-radius: 4px; + text-align: left; +} + +.admonition.note { /* csslint allow: adjoining-classes */ + color: #3a87ad; + background-color: #d9edf7; + border-color: #bce8f1; +} + +.admonition.warning { /* csslint allow: adjoining-classes */ + color: #c09853; + background-color: #fcf8e3; + border-color: #fbeed5; +} + +.admonition.danger { /* csslint allow: adjoining-classes */ + color: #b94a48; + background-color: #f2dede; + border-color: #eed3d7; +} + +.admonition-title { + font-weight: bold; + text-align: left; +} + +@media (max-width: 991.98px) { + .navbar-collapse.show { /* csslint allow: adjoining-classes */ + overflow-y: auto; + max-height: calc(100vh - 3.5rem); + } +} + +.dropdown-item.open { /* csslint allow: adjoining-classes */ + color: #fff; + background-color: #2FA4E7; +} + +.dropdown-submenu > .dropdown-menu { + margin: 0 0 0 1.5rem; + padding: 0; + border-width: 0; +} + +.dropdown-submenu > a::after { + display: block; + content: " "; + float: right; + width: 0; + height: 0; + border-color: transparent; + border-style: solid; + border-width: 5px 0 5px 5px; + border-left-color: #ccc; + margin-top: 5px; + margin-right: -10px; +} + +.dropdown-submenu:hover > a::after { + border-left-color: #fff; +} + +@media (min-width: 992px) { + .dropdown-menu { + overflow-y: auto; + max-height: calc(100vh - 3.5rem); + } + + .dropdown-submenu { + position: relative; + } + + .dropdown-submenu > .dropdown-menu { + /* csslint ignore:start */ + position: fixed !important; + /* csslint ignore:end */ + margin-top: -9px; + margin-left: -2px; + border-width: 1px; + padding: 0.5rem 0; + } + + .dropdown-submenu.pull-left { /* csslint allow: adjoining-classes */ + float: none; + } + + .dropdown-submenu.pull-left > .dropdown-menu { /* csslint allow: adjoining-classes */ + left: -100%; + margin-left: 10px; + } +} + +@media print { + /* Remove sidebar when print */ + .col-md-3 { display: none; } +} diff --git a/docs/site/css/bootstrap.min.css b/docs/site/css/bootstrap.min.css new file mode 100644 index 000000000..4ce503d7f --- /dev/null +++ b/docs/site/css/bootstrap.min.css @@ -0,0 +1,12 @@ +/*! + * Bootswatch v4.1.3 + * Homepage: https://bootswatch.com + * Copyright 2012-2018 Thomas Park + * Licensed under MIT + * Based on Bootstrap +*//*! + * Bootstrap v4.1.3 (https://getbootstrap.com/) + * Copyright 2011-2018 The Bootstrap Authors + * Copyright 2011-2018 Twitter, Inc. + * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) + */:root{--blue:#033C73;--indigo:#6610f2;--purple:#6f42c1;--pink:#e83e8c;--red:#C71C22;--orange:#fd7e14;--yellow:#DD5600;--green:#73A839;--teal:#20c997;--cyan:#2FA4E7;--white:#fff;--gray:#868e96;--gray-dark:#343a40;--primary:#2FA4E7;--secondary:#e9ecef;--success:#73A839;--info:#033C73;--warning:#DD5600;--danger:#C71C22;--light:#f8f9fa;--dark:#343a40;--breakpoint-xs:0;--breakpoint-sm:576px;--breakpoint-md:768px;--breakpoint-lg:992px;--breakpoint-xl:1200px;--font-family-sans-serif:-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji";--font-family-monospace:SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace}*,*::before,*::after{-webkit-box-sizing:border-box;box-sizing:border-box}html{font-family:sans-serif;line-height:1.15;-webkit-text-size-adjust:100%;-ms-text-size-adjust:100%;-ms-overflow-style:scrollbar;-webkit-tap-highlight-color:transparent}@-ms-viewport{width:device-width}article,aside,figcaption,figure,footer,header,hgroup,main,nav,section{display:block}body{margin:0;font-family:-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji";font-size:1rem;font-weight:400;line-height:1.5;color:#495057;text-align:left;background-color:#fff}[tabindex="-1"]:focus{outline:0 !important}hr{-webkit-box-sizing:content-box;box-sizing:content-box;height:0;overflow:visible}h1,h2,h3,h4,h5,h6{margin-top:0;margin-bottom:0.5rem}p{margin-top:0;margin-bottom:1rem}abbr[title],abbr[data-original-title]{text-decoration:underline;-webkit-text-decoration:underline dotted;text-decoration:underline dotted;cursor:help;border-bottom:0}address{margin-bottom:1rem;font-style:normal;line-height:inherit}ol,ul,dl{margin-top:0;margin-bottom:1rem}ol ol,ul ul,ol ul,ul ol{margin-bottom:0}dt{font-weight:700}dd{margin-bottom:.5rem;margin-left:0}blockquote{margin:0 0 1rem}dfn{font-style:italic}b,strong{font-weight:bolder}small{font-size:80%}sub,sup{position:relative;font-size:75%;line-height:0;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}a{color:#2FA4E7;text-decoration:none;background-color:transparent;-webkit-text-decoration-skip:objects}a:hover{color:#157ab5;text-decoration:underline}a:not([href]):not([tabindex]){color:inherit;text-decoration:none}a:not([href]):not([tabindex]):hover,a:not([href]):not([tabindex]):focus{color:inherit;text-decoration:none}a:not([href]):not([tabindex]):focus{outline:0}pre,code,kbd,samp{font-family:SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;font-size:1em}pre{margin-top:0;margin-bottom:1rem;overflow:auto;-ms-overflow-style:scrollbar}figure{margin:0 0 1rem}img{vertical-align:middle;border-style:none}svg{overflow:hidden;vertical-align:middle}table{border-collapse:collapse}caption{padding-top:0.75rem;padding-bottom:0.75rem;color:#868e96;text-align:left;caption-side:bottom}th{text-align:inherit}label{display:inline-block;margin-bottom:0.5rem}button{border-radius:0}button:focus{outline:1px dotted;outline:5px auto -webkit-focus-ring-color}input,button,select,optgroup,textarea{margin:0;font-family:inherit;font-size:inherit;line-height:inherit}button,input{overflow:visible}button,select{text-transform:none}button,html [type="button"],[type="reset"],[type="submit"]{-webkit-appearance:button}button::-moz-focus-inner,[type="button"]::-moz-focus-inner,[type="reset"]::-moz-focus-inner,[type="submit"]::-moz-focus-inner{padding:0;border-style:none}input[type="radio"],input[type="checkbox"]{-webkit-box-sizing:border-box;box-sizing:border-box;padding:0}input[type="date"],input[type="time"],input[type="datetime-local"],input[type="month"]{-webkit-appearance:listbox}textarea{overflow:auto;resize:vertical}fieldset{min-width:0;padding:0;margin:0;border:0}legend{display:block;width:100%;max-width:100%;padding:0;margin-bottom:.5rem;font-size:1.5rem;line-height:inherit;color:inherit;white-space:normal}progress{vertical-align:baseline}[type="number"]::-webkit-inner-spin-button,[type="number"]::-webkit-outer-spin-button{height:auto}[type="search"]{outline-offset:-2px;-webkit-appearance:none}[type="search"]::-webkit-search-cancel-button,[type="search"]::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{font:inherit;-webkit-appearance:button}output{display:inline-block}summary{display:list-item;cursor:pointer}template{display:none}[hidden]{display:none !important}h1,h2,h3,h4,h5,h6,.h1,.h2,.h3,.h4,.h5,.h6{margin-bottom:0.5rem;font-family:inherit;font-weight:500;line-height:1.2;color:#2FA4E7}h1,.h1{font-size:2.5rem}h2,.h2{font-size:2rem}h3,.h3{font-size:1.75rem}h4,.h4{font-size:1.5rem}h5,.h5{font-size:1.25rem}h6,.h6{font-size:1rem}.lead{font-size:1.25rem;font-weight:300}.display-1{font-size:6rem;font-weight:300;line-height:1.2}.display-2{font-size:5.5rem;font-weight:300;line-height:1.2}.display-3{font-size:4.5rem;font-weight:300;line-height:1.2}.display-4{font-size:3.5rem;font-weight:300;line-height:1.2}hr{margin-top:1rem;margin-bottom:1rem;border:0;border-top:1px solid rgba(0,0,0,0.1)}small,.small{font-size:80%;font-weight:400}mark,.mark{padding:0.2em;background-color:#fcf8e3}.list-unstyled{padding-left:0;list-style:none}.list-inline{padding-left:0;list-style:none}.list-inline-item{display:inline-block}.list-inline-item:not(:last-child){margin-right:0.5rem}.initialism{font-size:90%;text-transform:uppercase}.blockquote{margin-bottom:1rem;font-size:1.25rem}.blockquote-footer{display:block;font-size:80%;color:#868e96}.blockquote-footer::before{content:"\2014 \00A0"}.img-fluid{max-width:100%;height:auto}.img-thumbnail{padding:0.25rem;background-color:#fff;border:1px solid #dee2e6;border-radius:0.25rem;max-width:100%;height:auto}.figure{display:inline-block}.figure-img{margin-bottom:0.5rem;line-height:1}.figure-caption{font-size:90%;color:#868e96}code{font-size:87.5%;color:#e83e8c;word-break:break-word}a>code{color:inherit}kbd{padding:0.2rem 0.4rem;font-size:87.5%;color:#fff;background-color:#212529;border-radius:0.2rem}kbd kbd{padding:0;font-size:100%;font-weight:700}pre{display:block;font-size:87.5%;color:#212529}pre code{font-size:inherit;color:inherit;word-break:normal}.pre-scrollable{max-height:340px;overflow-y:scroll}.container{width:100%;padding-right:15px;padding-left:15px;margin-right:auto;margin-left:auto}@media (min-width: 576px){.container{max-width:540px}}@media (min-width: 768px){.container{max-width:720px}}@media (min-width: 992px){.container{max-width:960px}}@media (min-width: 1200px){.container{max-width:1140px}}.container-fluid{width:100%;padding-right:15px;padding-left:15px;margin-right:auto;margin-left:auto}.row{display:-webkit-box;display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;margin-right:-15px;margin-left:-15px}.no-gutters{margin-right:0;margin-left:0}.no-gutters>.col,.no-gutters>[class*="col-"]{padding-right:0;padding-left:0}.col-1,.col-2,.col-3,.col-4,.col-5,.col-6,.col-7,.col-8,.col-9,.col-10,.col-11,.col-12,.col,.col-auto,.col-sm-1,.col-sm-2,.col-sm-3,.col-sm-4,.col-sm-5,.col-sm-6,.col-sm-7,.col-sm-8,.col-sm-9,.col-sm-10,.col-sm-11,.col-sm-12,.col-sm,.col-sm-auto,.col-md-1,.col-md-2,.col-md-3,.col-md-4,.col-md-5,.col-md-6,.col-md-7,.col-md-8,.col-md-9,.col-md-10,.col-md-11,.col-md-12,.col-md,.col-md-auto,.col-lg-1,.col-lg-2,.col-lg-3,.col-lg-4,.col-lg-5,.col-lg-6,.col-lg-7,.col-lg-8,.col-lg-9,.col-lg-10,.col-lg-11,.col-lg-12,.col-lg,.col-lg-auto,.col-xl-1,.col-xl-2,.col-xl-3,.col-xl-4,.col-xl-5,.col-xl-6,.col-xl-7,.col-xl-8,.col-xl-9,.col-xl-10,.col-xl-11,.col-xl-12,.col-xl,.col-xl-auto{position:relative;width:100%;min-height:1px;padding-right:15px;padding-left:15px}.col{-ms-flex-preferred-size:0;flex-basis:0;-webkit-box-flex:1;-ms-flex-positive:1;flex-grow:1;max-width:100%}.col-auto{-webkit-box-flex:0;-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:none}.col-1{-webkit-box-flex:0;-ms-flex:0 0 8.3333333333%;flex:0 0 8.3333333333%;max-width:8.3333333333%}.col-2{-webkit-box-flex:0;-ms-flex:0 0 16.6666666667%;flex:0 0 16.6666666667%;max-width:16.6666666667%}.col-3{-webkit-box-flex:0;-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-4{-webkit-box-flex:0;-ms-flex:0 0 33.3333333333%;flex:0 0 33.3333333333%;max-width:33.3333333333%}.col-5{-webkit-box-flex:0;-ms-flex:0 0 41.6666666667%;flex:0 0 41.6666666667%;max-width:41.6666666667%}.col-6{-webkit-box-flex:0;-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-7{-webkit-box-flex:0;-ms-flex:0 0 58.3333333333%;flex:0 0 58.3333333333%;max-width:58.3333333333%}.col-8{-webkit-box-flex:0;-ms-flex:0 0 66.6666666667%;flex:0 0 66.6666666667%;max-width:66.6666666667%}.col-9{-webkit-box-flex:0;-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-10{-webkit-box-flex:0;-ms-flex:0 0 83.3333333333%;flex:0 0 83.3333333333%;max-width:83.3333333333%}.col-11{-webkit-box-flex:0;-ms-flex:0 0 91.6666666667%;flex:0 0 91.6666666667%;max-width:91.6666666667%}.col-12{-webkit-box-flex:0;-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-first{-webkit-box-ordinal-group:0;-ms-flex-order:-1;order:-1}.order-last{-webkit-box-ordinal-group:14;-ms-flex-order:13;order:13}.order-0{-webkit-box-ordinal-group:1;-ms-flex-order:0;order:0}.order-1{-webkit-box-ordinal-group:2;-ms-flex-order:1;order:1}.order-2{-webkit-box-ordinal-group:3;-ms-flex-order:2;order:2}.order-3{-webkit-box-ordinal-group:4;-ms-flex-order:3;order:3}.order-4{-webkit-box-ordinal-group:5;-ms-flex-order:4;order:4}.order-5{-webkit-box-ordinal-group:6;-ms-flex-order:5;order:5}.order-6{-webkit-box-ordinal-group:7;-ms-flex-order:6;order:6}.order-7{-webkit-box-ordinal-group:8;-ms-flex-order:7;order:7}.order-8{-webkit-box-ordinal-group:9;-ms-flex-order:8;order:8}.order-9{-webkit-box-ordinal-group:10;-ms-flex-order:9;order:9}.order-10{-webkit-box-ordinal-group:11;-ms-flex-order:10;order:10}.order-11{-webkit-box-ordinal-group:12;-ms-flex-order:11;order:11}.order-12{-webkit-box-ordinal-group:13;-ms-flex-order:12;order:12}.offset-1{margin-left:8.3333333333%}.offset-2{margin-left:16.6666666667%}.offset-3{margin-left:25%}.offset-4{margin-left:33.3333333333%}.offset-5{margin-left:41.6666666667%}.offset-6{margin-left:50%}.offset-7{margin-left:58.3333333333%}.offset-8{margin-left:66.6666666667%}.offset-9{margin-left:75%}.offset-10{margin-left:83.3333333333%}.offset-11{margin-left:91.6666666667%}@media (min-width: 576px){.col-sm{-ms-flex-preferred-size:0;flex-basis:0;-webkit-box-flex:1;-ms-flex-positive:1;flex-grow:1;max-width:100%}.col-sm-auto{-webkit-box-flex:0;-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:none}.col-sm-1{-webkit-box-flex:0;-ms-flex:0 0 8.3333333333%;flex:0 0 8.3333333333%;max-width:8.3333333333%}.col-sm-2{-webkit-box-flex:0;-ms-flex:0 0 16.6666666667%;flex:0 0 16.6666666667%;max-width:16.6666666667%}.col-sm-3{-webkit-box-flex:0;-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-sm-4{-webkit-box-flex:0;-ms-flex:0 0 33.3333333333%;flex:0 0 33.3333333333%;max-width:33.3333333333%}.col-sm-5{-webkit-box-flex:0;-ms-flex:0 0 41.6666666667%;flex:0 0 41.6666666667%;max-width:41.6666666667%}.col-sm-6{-webkit-box-flex:0;-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-sm-7{-webkit-box-flex:0;-ms-flex:0 0 58.3333333333%;flex:0 0 58.3333333333%;max-width:58.3333333333%}.col-sm-8{-webkit-box-flex:0;-ms-flex:0 0 66.6666666667%;flex:0 0 66.6666666667%;max-width:66.6666666667%}.col-sm-9{-webkit-box-flex:0;-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-sm-10{-webkit-box-flex:0;-ms-flex:0 0 83.3333333333%;flex:0 0 83.3333333333%;max-width:83.3333333333%}.col-sm-11{-webkit-box-flex:0;-ms-flex:0 0 91.6666666667%;flex:0 0 91.6666666667%;max-width:91.6666666667%}.col-sm-12{-webkit-box-flex:0;-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-sm-first{-webkit-box-ordinal-group:0;-ms-flex-order:-1;order:-1}.order-sm-last{-webkit-box-ordinal-group:14;-ms-flex-order:13;order:13}.order-sm-0{-webkit-box-ordinal-group:1;-ms-flex-order:0;order:0}.order-sm-1{-webkit-box-ordinal-group:2;-ms-flex-order:1;order:1}.order-sm-2{-webkit-box-ordinal-group:3;-ms-flex-order:2;order:2}.order-sm-3{-webkit-box-ordinal-group:4;-ms-flex-order:3;order:3}.order-sm-4{-webkit-box-ordinal-group:5;-ms-flex-order:4;order:4}.order-sm-5{-webkit-box-ordinal-group:6;-ms-flex-order:5;order:5}.order-sm-6{-webkit-box-ordinal-group:7;-ms-flex-order:6;order:6}.order-sm-7{-webkit-box-ordinal-group:8;-ms-flex-order:7;order:7}.order-sm-8{-webkit-box-ordinal-group:9;-ms-flex-order:8;order:8}.order-sm-9{-webkit-box-ordinal-group:10;-ms-flex-order:9;order:9}.order-sm-10{-webkit-box-ordinal-group:11;-ms-flex-order:10;order:10}.order-sm-11{-webkit-box-ordinal-group:12;-ms-flex-order:11;order:11}.order-sm-12{-webkit-box-ordinal-group:13;-ms-flex-order:12;order:12}.offset-sm-0{margin-left:0}.offset-sm-1{margin-left:8.3333333333%}.offset-sm-2{margin-left:16.6666666667%}.offset-sm-3{margin-left:25%}.offset-sm-4{margin-left:33.3333333333%}.offset-sm-5{margin-left:41.6666666667%}.offset-sm-6{margin-left:50%}.offset-sm-7{margin-left:58.3333333333%}.offset-sm-8{margin-left:66.6666666667%}.offset-sm-9{margin-left:75%}.offset-sm-10{margin-left:83.3333333333%}.offset-sm-11{margin-left:91.6666666667%}}@media (min-width: 768px){.col-md{-ms-flex-preferred-size:0;flex-basis:0;-webkit-box-flex:1;-ms-flex-positive:1;flex-grow:1;max-width:100%}.col-md-auto{-webkit-box-flex:0;-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:none}.col-md-1{-webkit-box-flex:0;-ms-flex:0 0 8.3333333333%;flex:0 0 8.3333333333%;max-width:8.3333333333%}.col-md-2{-webkit-box-flex:0;-ms-flex:0 0 16.6666666667%;flex:0 0 16.6666666667%;max-width:16.6666666667%}.col-md-3{-webkit-box-flex:0;-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-md-4{-webkit-box-flex:0;-ms-flex:0 0 33.3333333333%;flex:0 0 33.3333333333%;max-width:33.3333333333%}.col-md-5{-webkit-box-flex:0;-ms-flex:0 0 41.6666666667%;flex:0 0 41.6666666667%;max-width:41.6666666667%}.col-md-6{-webkit-box-flex:0;-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-md-7{-webkit-box-flex:0;-ms-flex:0 0 58.3333333333%;flex:0 0 58.3333333333%;max-width:58.3333333333%}.col-md-8{-webkit-box-flex:0;-ms-flex:0 0 66.6666666667%;flex:0 0 66.6666666667%;max-width:66.6666666667%}.col-md-9{-webkit-box-flex:0;-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-md-10{-webkit-box-flex:0;-ms-flex:0 0 83.3333333333%;flex:0 0 83.3333333333%;max-width:83.3333333333%}.col-md-11{-webkit-box-flex:0;-ms-flex:0 0 91.6666666667%;flex:0 0 91.6666666667%;max-width:91.6666666667%}.col-md-12{-webkit-box-flex:0;-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-md-first{-webkit-box-ordinal-group:0;-ms-flex-order:-1;order:-1}.order-md-last{-webkit-box-ordinal-group:14;-ms-flex-order:13;order:13}.order-md-0{-webkit-box-ordinal-group:1;-ms-flex-order:0;order:0}.order-md-1{-webkit-box-ordinal-group:2;-ms-flex-order:1;order:1}.order-md-2{-webkit-box-ordinal-group:3;-ms-flex-order:2;order:2}.order-md-3{-webkit-box-ordinal-group:4;-ms-flex-order:3;order:3}.order-md-4{-webkit-box-ordinal-group:5;-ms-flex-order:4;order:4}.order-md-5{-webkit-box-ordinal-group:6;-ms-flex-order:5;order:5}.order-md-6{-webkit-box-ordinal-group:7;-ms-flex-order:6;order:6}.order-md-7{-webkit-box-ordinal-group:8;-ms-flex-order:7;order:7}.order-md-8{-webkit-box-ordinal-group:9;-ms-flex-order:8;order:8}.order-md-9{-webkit-box-ordinal-group:10;-ms-flex-order:9;order:9}.order-md-10{-webkit-box-ordinal-group:11;-ms-flex-order:10;order:10}.order-md-11{-webkit-box-ordinal-group:12;-ms-flex-order:11;order:11}.order-md-12{-webkit-box-ordinal-group:13;-ms-flex-order:12;order:12}.offset-md-0{margin-left:0}.offset-md-1{margin-left:8.3333333333%}.offset-md-2{margin-left:16.6666666667%}.offset-md-3{margin-left:25%}.offset-md-4{margin-left:33.3333333333%}.offset-md-5{margin-left:41.6666666667%}.offset-md-6{margin-left:50%}.offset-md-7{margin-left:58.3333333333%}.offset-md-8{margin-left:66.6666666667%}.offset-md-9{margin-left:75%}.offset-md-10{margin-left:83.3333333333%}.offset-md-11{margin-left:91.6666666667%}}@media (min-width: 992px){.col-lg{-ms-flex-preferred-size:0;flex-basis:0;-webkit-box-flex:1;-ms-flex-positive:1;flex-grow:1;max-width:100%}.col-lg-auto{-webkit-box-flex:0;-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:none}.col-lg-1{-webkit-box-flex:0;-ms-flex:0 0 8.3333333333%;flex:0 0 8.3333333333%;max-width:8.3333333333%}.col-lg-2{-webkit-box-flex:0;-ms-flex:0 0 16.6666666667%;flex:0 0 16.6666666667%;max-width:16.6666666667%}.col-lg-3{-webkit-box-flex:0;-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-lg-4{-webkit-box-flex:0;-ms-flex:0 0 33.3333333333%;flex:0 0 33.3333333333%;max-width:33.3333333333%}.col-lg-5{-webkit-box-flex:0;-ms-flex:0 0 41.6666666667%;flex:0 0 41.6666666667%;max-width:41.6666666667%}.col-lg-6{-webkit-box-flex:0;-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-lg-7{-webkit-box-flex:0;-ms-flex:0 0 58.3333333333%;flex:0 0 58.3333333333%;max-width:58.3333333333%}.col-lg-8{-webkit-box-flex:0;-ms-flex:0 0 66.6666666667%;flex:0 0 66.6666666667%;max-width:66.6666666667%}.col-lg-9{-webkit-box-flex:0;-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-lg-10{-webkit-box-flex:0;-ms-flex:0 0 83.3333333333%;flex:0 0 83.3333333333%;max-width:83.3333333333%}.col-lg-11{-webkit-box-flex:0;-ms-flex:0 0 91.6666666667%;flex:0 0 91.6666666667%;max-width:91.6666666667%}.col-lg-12{-webkit-box-flex:0;-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-lg-first{-webkit-box-ordinal-group:0;-ms-flex-order:-1;order:-1}.order-lg-last{-webkit-box-ordinal-group:14;-ms-flex-order:13;order:13}.order-lg-0{-webkit-box-ordinal-group:1;-ms-flex-order:0;order:0}.order-lg-1{-webkit-box-ordinal-group:2;-ms-flex-order:1;order:1}.order-lg-2{-webkit-box-ordinal-group:3;-ms-flex-order:2;order:2}.order-lg-3{-webkit-box-ordinal-group:4;-ms-flex-order:3;order:3}.order-lg-4{-webkit-box-ordinal-group:5;-ms-flex-order:4;order:4}.order-lg-5{-webkit-box-ordinal-group:6;-ms-flex-order:5;order:5}.order-lg-6{-webkit-box-ordinal-group:7;-ms-flex-order:6;order:6}.order-lg-7{-webkit-box-ordinal-group:8;-ms-flex-order:7;order:7}.order-lg-8{-webkit-box-ordinal-group:9;-ms-flex-order:8;order:8}.order-lg-9{-webkit-box-ordinal-group:10;-ms-flex-order:9;order:9}.order-lg-10{-webkit-box-ordinal-group:11;-ms-flex-order:10;order:10}.order-lg-11{-webkit-box-ordinal-group:12;-ms-flex-order:11;order:11}.order-lg-12{-webkit-box-ordinal-group:13;-ms-flex-order:12;order:12}.offset-lg-0{margin-left:0}.offset-lg-1{margin-left:8.3333333333%}.offset-lg-2{margin-left:16.6666666667%}.offset-lg-3{margin-left:25%}.offset-lg-4{margin-left:33.3333333333%}.offset-lg-5{margin-left:41.6666666667%}.offset-lg-6{margin-left:50%}.offset-lg-7{margin-left:58.3333333333%}.offset-lg-8{margin-left:66.6666666667%}.offset-lg-9{margin-left:75%}.offset-lg-10{margin-left:83.3333333333%}.offset-lg-11{margin-left:91.6666666667%}}@media (min-width: 1200px){.col-xl{-ms-flex-preferred-size:0;flex-basis:0;-webkit-box-flex:1;-ms-flex-positive:1;flex-grow:1;max-width:100%}.col-xl-auto{-webkit-box-flex:0;-ms-flex:0 0 auto;flex:0 0 auto;width:auto;max-width:none}.col-xl-1{-webkit-box-flex:0;-ms-flex:0 0 8.3333333333%;flex:0 0 8.3333333333%;max-width:8.3333333333%}.col-xl-2{-webkit-box-flex:0;-ms-flex:0 0 16.6666666667%;flex:0 0 16.6666666667%;max-width:16.6666666667%}.col-xl-3{-webkit-box-flex:0;-ms-flex:0 0 25%;flex:0 0 25%;max-width:25%}.col-xl-4{-webkit-box-flex:0;-ms-flex:0 0 33.3333333333%;flex:0 0 33.3333333333%;max-width:33.3333333333%}.col-xl-5{-webkit-box-flex:0;-ms-flex:0 0 41.6666666667%;flex:0 0 41.6666666667%;max-width:41.6666666667%}.col-xl-6{-webkit-box-flex:0;-ms-flex:0 0 50%;flex:0 0 50%;max-width:50%}.col-xl-7{-webkit-box-flex:0;-ms-flex:0 0 58.3333333333%;flex:0 0 58.3333333333%;max-width:58.3333333333%}.col-xl-8{-webkit-box-flex:0;-ms-flex:0 0 66.6666666667%;flex:0 0 66.6666666667%;max-width:66.6666666667%}.col-xl-9{-webkit-box-flex:0;-ms-flex:0 0 75%;flex:0 0 75%;max-width:75%}.col-xl-10{-webkit-box-flex:0;-ms-flex:0 0 83.3333333333%;flex:0 0 83.3333333333%;max-width:83.3333333333%}.col-xl-11{-webkit-box-flex:0;-ms-flex:0 0 91.6666666667%;flex:0 0 91.6666666667%;max-width:91.6666666667%}.col-xl-12{-webkit-box-flex:0;-ms-flex:0 0 100%;flex:0 0 100%;max-width:100%}.order-xl-first{-webkit-box-ordinal-group:0;-ms-flex-order:-1;order:-1}.order-xl-last{-webkit-box-ordinal-group:14;-ms-flex-order:13;order:13}.order-xl-0{-webkit-box-ordinal-group:1;-ms-flex-order:0;order:0}.order-xl-1{-webkit-box-ordinal-group:2;-ms-flex-order:1;order:1}.order-xl-2{-webkit-box-ordinal-group:3;-ms-flex-order:2;order:2}.order-xl-3{-webkit-box-ordinal-group:4;-ms-flex-order:3;order:3}.order-xl-4{-webkit-box-ordinal-group:5;-ms-flex-order:4;order:4}.order-xl-5{-webkit-box-ordinal-group:6;-ms-flex-order:5;order:5}.order-xl-6{-webkit-box-ordinal-group:7;-ms-flex-order:6;order:6}.order-xl-7{-webkit-box-ordinal-group:8;-ms-flex-order:7;order:7}.order-xl-8{-webkit-box-ordinal-group:9;-ms-flex-order:8;order:8}.order-xl-9{-webkit-box-ordinal-group:10;-ms-flex-order:9;order:9}.order-xl-10{-webkit-box-ordinal-group:11;-ms-flex-order:10;order:10}.order-xl-11{-webkit-box-ordinal-group:12;-ms-flex-order:11;order:11}.order-xl-12{-webkit-box-ordinal-group:13;-ms-flex-order:12;order:12}.offset-xl-0{margin-left:0}.offset-xl-1{margin-left:8.3333333333%}.offset-xl-2{margin-left:16.6666666667%}.offset-xl-3{margin-left:25%}.offset-xl-4{margin-left:33.3333333333%}.offset-xl-5{margin-left:41.6666666667%}.offset-xl-6{margin-left:50%}.offset-xl-7{margin-left:58.3333333333%}.offset-xl-8{margin-left:66.6666666667%}.offset-xl-9{margin-left:75%}.offset-xl-10{margin-left:83.3333333333%}.offset-xl-11{margin-left:91.6666666667%}}.table{width:100%;margin-bottom:1rem;background-color:transparent}.table th,.table td{padding:0.75rem;vertical-align:top;border-top:1px solid #dee2e6}.table thead th{vertical-align:bottom;border-bottom:2px solid #dee2e6}.table tbody+tbody{border-top:2px solid #dee2e6}.table .table{background-color:#fff}.table-sm th,.table-sm td{padding:0.3rem}.table-bordered{border:1px solid #dee2e6}.table-bordered th,.table-bordered td{border:1px solid #dee2e6}.table-bordered thead th,.table-bordered thead td{border-bottom-width:2px}.table-borderless th,.table-borderless td,.table-borderless thead th,.table-borderless tbody+tbody{border:0}.table-striped tbody tr:nth-of-type(odd){background-color:rgba(0,0,0,0.05)}.table-hover tbody tr:hover{background-color:rgba(0,0,0,0.075)}.table-primary,.table-primary>th,.table-primary>td{background-color:#c5e6f8}.table-hover .table-primary:hover{background-color:#aedcf5}.table-hover .table-primary:hover>td,.table-hover .table-primary:hover>th{background-color:#aedcf5}.table-secondary,.table-secondary>th,.table-secondary>td{background-color:#f9fafb}.table-hover .table-secondary:hover{background-color:#eaedf1}.table-hover .table-secondary:hover>td,.table-hover .table-secondary:hover>th{background-color:#eaedf1}.table-success,.table-success>th,.table-success>td{background-color:#d8e7c8}.table-hover .table-success:hover{background-color:#cbdfb6}.table-hover .table-success:hover>td,.table-hover .table-success:hover>th{background-color:#cbdfb6}.table-info,.table-info>th,.table-info>td{background-color:#b8c8d8}.table-hover .table-info:hover{background-color:#a8bbcf}.table-hover .table-info:hover>td,.table-hover .table-info:hover>th{background-color:#a8bbcf}.table-warning,.table-warning>th,.table-warning>td{background-color:#f5d0b8}.table-hover .table-warning:hover{background-color:#f2c1a2}.table-hover .table-warning:hover>td,.table-hover .table-warning:hover>th{background-color:#f2c1a2}.table-danger,.table-danger>th,.table-danger>td{background-color:#efbfc1}.table-hover .table-danger:hover{background-color:#eaabad}.table-hover .table-danger:hover>td,.table-hover .table-danger:hover>th{background-color:#eaabad}.table-light,.table-light>th,.table-light>td{background-color:#fdfdfe}.table-hover .table-light:hover{background-color:#ececf6}.table-hover .table-light:hover>td,.table-hover .table-light:hover>th{background-color:#ececf6}.table-dark,.table-dark>th,.table-dark>td{background-color:#c6c8ca}.table-hover .table-dark:hover{background-color:#b9bbbe}.table-hover .table-dark:hover>td,.table-hover .table-dark:hover>th{background-color:#b9bbbe}.table-active,.table-active>th,.table-active>td{background-color:rgba(0,0,0,0.075)}.table-hover .table-active:hover{background-color:rgba(0,0,0,0.075)}.table-hover .table-active:hover>td,.table-hover .table-active:hover>th{background-color:rgba(0,0,0,0.075)}.table .thead-dark th{color:#fff;background-color:#212529;border-color:#32383e}.table .thead-light th{color:#495057;background-color:#e9ecef;border-color:#dee2e6}.table-dark{color:#fff;background-color:#212529}.table-dark th,.table-dark td,.table-dark thead th{border-color:#32383e}.table-dark.table-bordered{border:0}.table-dark.table-striped tbody tr:nth-of-type(odd){background-color:rgba(255,255,255,0.05)}.table-dark.table-hover tbody tr:hover{background-color:rgba(255,255,255,0.075)}@media (max-width: 575.98px){.table-responsive-sm{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch;-ms-overflow-style:-ms-autohiding-scrollbar}.table-responsive-sm>.table-bordered{border:0}}@media (max-width: 767.98px){.table-responsive-md{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch;-ms-overflow-style:-ms-autohiding-scrollbar}.table-responsive-md>.table-bordered{border:0}}@media (max-width: 991.98px){.table-responsive-lg{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch;-ms-overflow-style:-ms-autohiding-scrollbar}.table-responsive-lg>.table-bordered{border:0}}@media (max-width: 1199.98px){.table-responsive-xl{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch;-ms-overflow-style:-ms-autohiding-scrollbar}.table-responsive-xl>.table-bordered{border:0}}.table-responsive{display:block;width:100%;overflow-x:auto;-webkit-overflow-scrolling:touch;-ms-overflow-style:-ms-autohiding-scrollbar}.table-responsive>.table-bordered{border:0}.form-control{display:block;width:100%;height:calc(2.25rem + 2px);padding:0.375rem 0.75rem;font-size:1rem;line-height:1.5;color:#495057;background-color:#fff;background-clip:padding-box;border:1px solid #ced4da;border-radius:0.25rem;-webkit-transition:border-color 0.15s ease-in-out, -webkit-box-shadow 0.15s ease-in-out;transition:border-color 0.15s ease-in-out, -webkit-box-shadow 0.15s ease-in-out;transition:border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out;transition:border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out, -webkit-box-shadow 0.15s ease-in-out}@media screen and (prefers-reduced-motion: reduce){.form-control{-webkit-transition:none;transition:none}}.form-control::-ms-expand{background-color:transparent;border:0}.form-control:focus{color:#495057;background-color:#fff;border-color:#a1d6f4;outline:0;-webkit-box-shadow:0 0 0 0.2rem rgba(47,164,231,0.25);box-shadow:0 0 0 0.2rem rgba(47,164,231,0.25)}.form-control::-webkit-input-placeholder{color:#868e96;opacity:1}.form-control:-ms-input-placeholder{color:#868e96;opacity:1}.form-control::-ms-input-placeholder{color:#868e96;opacity:1}.form-control::placeholder{color:#868e96;opacity:1}.form-control:disabled,.form-control[readonly]{background-color:#e9ecef;opacity:1}select.form-control:focus::-ms-value{color:#495057;background-color:#fff}.form-control-file,.form-control-range{display:block;width:100%}.col-form-label{padding-top:calc(0.375rem + 1px);padding-bottom:calc(0.375rem + 1px);margin-bottom:0;font-size:inherit;line-height:1.5}.col-form-label-lg{padding-top:calc(0.5rem + 1px);padding-bottom:calc(0.5rem + 1px);font-size:1.25rem;line-height:1.5}.col-form-label-sm{padding-top:calc(0.25rem + 1px);padding-bottom:calc(0.25rem + 1px);font-size:0.875rem;line-height:1.5}.form-control-plaintext{display:block;width:100%;padding-top:0.375rem;padding-bottom:0.375rem;margin-bottom:0;line-height:1.5;color:#495057;background-color:transparent;border:solid transparent;border-width:1px 0}.form-control-plaintext.form-control-sm,.form-control-plaintext.form-control-lg{padding-right:0;padding-left:0}.form-control-sm{height:calc(1.8125rem + 2px);padding:0.25rem 0.5rem;font-size:0.875rem;line-height:1.5;border-radius:0.2rem}.form-control-lg{height:calc(2.875rem + 2px);padding:0.5rem 1rem;font-size:1.25rem;line-height:1.5;border-radius:0.3rem}select.form-control[size],select.form-control[multiple]{height:auto}textarea.form-control{height:auto}.form-group{margin-bottom:1rem}.form-text{display:block;margin-top:0.25rem}.form-row{display:-webkit-box;display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;margin-right:-5px;margin-left:-5px}.form-row>.col,.form-row>[class*="col-"]{padding-right:5px;padding-left:5px}.form-check{position:relative;display:block;padding-left:1.25rem}.form-check-input{position:absolute;margin-top:0.3rem;margin-left:-1.25rem}.form-check-input:disabled ~ .form-check-label{color:#868e96}.form-check-label{margin-bottom:0}.form-check-inline{display:-webkit-inline-box;display:-ms-inline-flexbox;display:inline-flex;-webkit-box-align:center;-ms-flex-align:center;align-items:center;padding-left:0;margin-right:0.75rem}.form-check-inline .form-check-input{position:static;margin-top:0;margin-right:0.3125rem;margin-left:0}.valid-feedback{display:none;width:100%;margin-top:0.25rem;font-size:80%;color:#73A839}.valid-tooltip{position:absolute;top:100%;z-index:5;display:none;max-width:100%;padding:0.25rem 0.5rem;margin-top:.1rem;font-size:0.875rem;line-height:1.5;color:#fff;background-color:rgba(115,168,57,0.9);border-radius:0.25rem}.was-validated .form-control:valid,.form-control.is-valid,.was-validated .custom-select:valid,.custom-select.is-valid{border-color:#73A839}.was-validated .form-control:valid:focus,.form-control.is-valid:focus,.was-validated .custom-select:valid:focus,.custom-select.is-valid:focus{border-color:#73A839;-webkit-box-shadow:0 0 0 0.2rem rgba(115,168,57,0.25);box-shadow:0 0 0 0.2rem rgba(115,168,57,0.25)}.was-validated .form-control:valid ~ .valid-feedback,.was-validated .form-control:valid ~ .valid-tooltip,.form-control.is-valid ~ .valid-feedback,.form-control.is-valid ~ .valid-tooltip,.was-validated .custom-select:valid ~ .valid-feedback,.was-validated .custom-select:valid ~ .valid-tooltip,.custom-select.is-valid ~ .valid-feedback,.custom-select.is-valid ~ .valid-tooltip{display:block}.was-validated .form-control-file:valid ~ .valid-feedback,.was-validated .form-control-file:valid ~ .valid-tooltip,.form-control-file.is-valid ~ .valid-feedback,.form-control-file.is-valid ~ .valid-tooltip{display:block}.was-validated .form-check-input:valid ~ .form-check-label,.form-check-input.is-valid ~ .form-check-label{color:#73A839}.was-validated .form-check-input:valid ~ .valid-feedback,.was-validated .form-check-input:valid ~ .valid-tooltip,.form-check-input.is-valid ~ .valid-feedback,.form-check-input.is-valid ~ .valid-tooltip{display:block}.was-validated .custom-control-input:valid ~ .custom-control-label,.custom-control-input.is-valid ~ .custom-control-label{color:#73A839}.was-validated .custom-control-input:valid ~ .custom-control-label::before,.custom-control-input.is-valid ~ .custom-control-label::before{background-color:#b2d789}.was-validated .custom-control-input:valid ~ .valid-feedback,.was-validated .custom-control-input:valid ~ .valid-tooltip,.custom-control-input.is-valid ~ .valid-feedback,.custom-control-input.is-valid ~ .valid-tooltip{display:block}.was-validated .custom-control-input:valid:checked ~ .custom-control-label::before,.custom-control-input.is-valid:checked ~ .custom-control-label::before{background-color:#8dc450}.was-validated .custom-control-input:valid:focus ~ .custom-control-label::before,.custom-control-input.is-valid:focus ~ .custom-control-label::before{-webkit-box-shadow:0 0 0 1px #fff,0 0 0 0.2rem rgba(115,168,57,0.25);box-shadow:0 0 0 1px #fff,0 0 0 0.2rem rgba(115,168,57,0.25)}.was-validated .custom-file-input:valid ~ .custom-file-label,.custom-file-input.is-valid ~ .custom-file-label{border-color:#73A839}.was-validated .custom-file-input:valid ~ .custom-file-label::after,.custom-file-input.is-valid ~ .custom-file-label::after{border-color:inherit}.was-validated .custom-file-input:valid ~ .valid-feedback,.was-validated .custom-file-input:valid ~ .valid-tooltip,.custom-file-input.is-valid ~ .valid-feedback,.custom-file-input.is-valid ~ .valid-tooltip{display:block}.was-validated .custom-file-input:valid:focus ~ .custom-file-label,.custom-file-input.is-valid:focus ~ .custom-file-label{-webkit-box-shadow:0 0 0 0.2rem rgba(115,168,57,0.25);box-shadow:0 0 0 0.2rem rgba(115,168,57,0.25)}.invalid-feedback{display:none;width:100%;margin-top:0.25rem;font-size:80%;color:#C71C22}.invalid-tooltip{position:absolute;top:100%;z-index:5;display:none;max-width:100%;padding:0.25rem 0.5rem;margin-top:.1rem;font-size:0.875rem;line-height:1.5;color:#fff;background-color:rgba(199,28,34,0.9);border-radius:0.25rem}.was-validated .form-control:invalid,.form-control.is-invalid,.was-validated .custom-select:invalid,.custom-select.is-invalid{border-color:#C71C22}.was-validated .form-control:invalid:focus,.form-control.is-invalid:focus,.was-validated .custom-select:invalid:focus,.custom-select.is-invalid:focus{border-color:#C71C22;-webkit-box-shadow:0 0 0 0.2rem rgba(199,28,34,0.25);box-shadow:0 0 0 0.2rem rgba(199,28,34,0.25)}.was-validated .form-control:invalid ~ .invalid-feedback,.was-validated .form-control:invalid ~ .invalid-tooltip,.form-control.is-invalid ~ .invalid-feedback,.form-control.is-invalid ~ .invalid-tooltip,.was-validated .custom-select:invalid ~ .invalid-feedback,.was-validated .custom-select:invalid ~ .invalid-tooltip,.custom-select.is-invalid ~ .invalid-feedback,.custom-select.is-invalid ~ .invalid-tooltip{display:block}.was-validated .form-control-file:invalid ~ .invalid-feedback,.was-validated .form-control-file:invalid ~ .invalid-tooltip,.form-control-file.is-invalid ~ .invalid-feedback,.form-control-file.is-invalid ~ .invalid-tooltip{display:block}.was-validated .form-check-input:invalid ~ .form-check-label,.form-check-input.is-invalid ~ .form-check-label{color:#C71C22}.was-validated .form-check-input:invalid ~ .invalid-feedback,.was-validated .form-check-input:invalid ~ .invalid-tooltip,.form-check-input.is-invalid ~ .invalid-feedback,.form-check-input.is-invalid ~ .invalid-tooltip{display:block}.was-validated .custom-control-input:invalid ~ .custom-control-label,.custom-control-input.is-invalid ~ .custom-control-label{color:#C71C22}.was-validated .custom-control-input:invalid ~ .custom-control-label::before,.custom-control-input.is-invalid ~ .custom-control-label::before{background-color:#ec777b}.was-validated .custom-control-input:invalid ~ .invalid-feedback,.was-validated .custom-control-input:invalid ~ .invalid-tooltip,.custom-control-input.is-invalid ~ .invalid-feedback,.custom-control-input.is-invalid ~ .invalid-tooltip{display:block}.was-validated .custom-control-input:invalid:checked ~ .custom-control-label::before,.custom-control-input.is-invalid:checked ~ .custom-control-label::before{background-color:#e2343a}.was-validated .custom-control-input:invalid:focus ~ .custom-control-label::before,.custom-control-input.is-invalid:focus ~ .custom-control-label::before{-webkit-box-shadow:0 0 0 1px #fff,0 0 0 0.2rem rgba(199,28,34,0.25);box-shadow:0 0 0 1px #fff,0 0 0 0.2rem rgba(199,28,34,0.25)}.was-validated .custom-file-input:invalid ~ .custom-file-label,.custom-file-input.is-invalid ~ .custom-file-label{border-color:#C71C22}.was-validated .custom-file-input:invalid ~ .custom-file-label::after,.custom-file-input.is-invalid ~ .custom-file-label::after{border-color:inherit}.was-validated .custom-file-input:invalid ~ .invalid-feedback,.was-validated .custom-file-input:invalid ~ .invalid-tooltip,.custom-file-input.is-invalid ~ .invalid-feedback,.custom-file-input.is-invalid ~ .invalid-tooltip{display:block}.was-validated .custom-file-input:invalid:focus ~ .custom-file-label,.custom-file-input.is-invalid:focus ~ .custom-file-label{-webkit-box-shadow:0 0 0 0.2rem rgba(199,28,34,0.25);box-shadow:0 0 0 0.2rem rgba(199,28,34,0.25)}.form-inline{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-flow:row wrap;flex-flow:row wrap;-webkit-box-align:center;-ms-flex-align:center;align-items:center}.form-inline .form-check{width:100%}@media (min-width: 576px){.form-inline label{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-align:center;-ms-flex-align:center;align-items:center;-webkit-box-pack:center;-ms-flex-pack:center;justify-content:center;margin-bottom:0}.form-inline .form-group{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-flex:0;-ms-flex:0 0 auto;flex:0 0 auto;-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-flow:row wrap;flex-flow:row wrap;-webkit-box-align:center;-ms-flex-align:center;align-items:center;margin-bottom:0}.form-inline .form-control{display:inline-block;width:auto;vertical-align:middle}.form-inline .form-control-plaintext{display:inline-block}.form-inline .input-group,.form-inline .custom-select{width:auto}.form-inline .form-check{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-align:center;-ms-flex-align:center;align-items:center;-webkit-box-pack:center;-ms-flex-pack:center;justify-content:center;width:auto;padding-left:0}.form-inline .form-check-input{position:relative;margin-top:0;margin-right:0.25rem;margin-left:0}.form-inline .custom-control{-webkit-box-align:center;-ms-flex-align:center;align-items:center;-webkit-box-pack:center;-ms-flex-pack:center;justify-content:center}.form-inline .custom-control-label{margin-bottom:0}}.btn{display:inline-block;font-weight:400;text-align:center;white-space:nowrap;vertical-align:middle;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;border:1px solid transparent;padding:0.375rem 0.75rem;font-size:1rem;line-height:1.5;border-radius:0.25rem;-webkit-transition:color 0.15s ease-in-out, background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, -webkit-box-shadow 0.15s ease-in-out;transition:color 0.15s ease-in-out, background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, -webkit-box-shadow 0.15s ease-in-out;transition:color 0.15s ease-in-out, background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out;transition:color 0.15s ease-in-out, background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out, -webkit-box-shadow 0.15s ease-in-out}@media screen and (prefers-reduced-motion: reduce){.btn{-webkit-transition:none;transition:none}}.btn:hover,.btn:focus{text-decoration:none}.btn:focus,.btn.focus{outline:0;-webkit-box-shadow:0 0 0 0.2rem rgba(47,164,231,0.25);box-shadow:0 0 0 0.2rem rgba(47,164,231,0.25)}.btn.disabled,.btn:disabled{opacity:0.65}.btn:not(:disabled):not(.disabled){cursor:pointer}a.btn.disabled,fieldset:disabled a.btn{pointer-events:none}.btn-primary{color:#fff;background-color:#2FA4E7;border-color:#2FA4E7}.btn-primary:hover{color:#fff;background-color:#1992d7;border-color:#178acc}.btn-primary:focus,.btn-primary.focus{-webkit-box-shadow:0 0 0 0.2rem rgba(47,164,231,0.5);box-shadow:0 0 0 0.2rem rgba(47,164,231,0.5)}.btn-primary.disabled,.btn-primary:disabled{color:#fff;background-color:#2FA4E7;border-color:#2FA4E7}.btn-primary:not(:disabled):not(.disabled):active,.btn-primary:not(:disabled):not(.disabled).active,.show>.btn-primary.dropdown-toggle{color:#fff;background-color:#178acc;border-color:#1682c0}.btn-primary:not(:disabled):not(.disabled):active:focus,.btn-primary:not(:disabled):not(.disabled).active:focus,.show>.btn-primary.dropdown-toggle:focus{-webkit-box-shadow:0 0 0 0.2rem rgba(47,164,231,0.5);box-shadow:0 0 0 0.2rem rgba(47,164,231,0.5)}.btn-secondary{color:#212529;background-color:#e9ecef;border-color:#e9ecef}.btn-secondary:hover{color:#212529;background-color:#d3d9df;border-color:#cbd3da}.btn-secondary:focus,.btn-secondary.focus{-webkit-box-shadow:0 0 0 0.2rem rgba(233,236,239,0.5);box-shadow:0 0 0 0.2rem rgba(233,236,239,0.5)}.btn-secondary.disabled,.btn-secondary:disabled{color:#212529;background-color:#e9ecef;border-color:#e9ecef}.btn-secondary:not(:disabled):not(.disabled):active,.btn-secondary:not(:disabled):not(.disabled).active,.show>.btn-secondary.dropdown-toggle{color:#212529;background-color:#cbd3da;border-color:#c4ccd4}.btn-secondary:not(:disabled):not(.disabled):active:focus,.btn-secondary:not(:disabled):not(.disabled).active:focus,.show>.btn-secondary.dropdown-toggle:focus{-webkit-box-shadow:0 0 0 0.2rem rgba(233,236,239,0.5);box-shadow:0 0 0 0.2rem rgba(233,236,239,0.5)}.btn-success{color:#fff;background-color:#73A839;border-color:#73A839}.btn-success:hover{color:#fff;background-color:#5f8b2f;border-color:#59822c}.btn-success:focus,.btn-success.focus{-webkit-box-shadow:0 0 0 0.2rem rgba(115,168,57,0.5);box-shadow:0 0 0 0.2rem rgba(115,168,57,0.5)}.btn-success.disabled,.btn-success:disabled{color:#fff;background-color:#73A839;border-color:#73A839}.btn-success:not(:disabled):not(.disabled):active,.btn-success:not(:disabled):not(.disabled).active,.show>.btn-success.dropdown-toggle{color:#fff;background-color:#59822c;border-color:#527829}.btn-success:not(:disabled):not(.disabled):active:focus,.btn-success:not(:disabled):not(.disabled).active:focus,.show>.btn-success.dropdown-toggle:focus{-webkit-box-shadow:0 0 0 0.2rem rgba(115,168,57,0.5);box-shadow:0 0 0 0.2rem rgba(115,168,57,0.5)}.btn-info{color:#fff;background-color:#033C73;border-color:#033C73}.btn-info:hover{color:#fff;background-color:#02294e;border-color:#022241}.btn-info:focus,.btn-info.focus{-webkit-box-shadow:0 0 0 0.2rem rgba(3,60,115,0.5);box-shadow:0 0 0 0.2rem rgba(3,60,115,0.5)}.btn-info.disabled,.btn-info:disabled{color:#fff;background-color:#033C73;border-color:#033C73}.btn-info:not(:disabled):not(.disabled):active,.btn-info:not(:disabled):not(.disabled).active,.show>.btn-info.dropdown-toggle{color:#fff;background-color:#022241;border-color:#011c35}.btn-info:not(:disabled):not(.disabled):active:focus,.btn-info:not(:disabled):not(.disabled).active:focus,.show>.btn-info.dropdown-toggle:focus{-webkit-box-shadow:0 0 0 0.2rem rgba(3,60,115,0.5);box-shadow:0 0 0 0.2rem rgba(3,60,115,0.5)}.btn-warning{color:#fff;background-color:#DD5600;border-color:#DD5600}.btn-warning:hover{color:#fff;background-color:#b74700;border-color:#aa4200}.btn-warning:focus,.btn-warning.focus{-webkit-box-shadow:0 0 0 0.2rem rgba(221,86,0,0.5);box-shadow:0 0 0 0.2rem rgba(221,86,0,0.5)}.btn-warning.disabled,.btn-warning:disabled{color:#fff;background-color:#DD5600;border-color:#DD5600}.btn-warning:not(:disabled):not(.disabled):active,.btn-warning:not(:disabled):not(.disabled).active,.show>.btn-warning.dropdown-toggle{color:#fff;background-color:#aa4200;border-color:#9d3d00}.btn-warning:not(:disabled):not(.disabled):active:focus,.btn-warning:not(:disabled):not(.disabled).active:focus,.show>.btn-warning.dropdown-toggle:focus{-webkit-box-shadow:0 0 0 0.2rem rgba(221,86,0,0.5);box-shadow:0 0 0 0.2rem rgba(221,86,0,0.5)}.btn-danger{color:#fff;background-color:#C71C22;border-color:#C71C22}.btn-danger:hover{color:#fff;background-color:#a5171c;border-color:#9a161a}.btn-danger:focus,.btn-danger.focus{-webkit-box-shadow:0 0 0 0.2rem rgba(199,28,34,0.5);box-shadow:0 0 0 0.2rem rgba(199,28,34,0.5)}.btn-danger.disabled,.btn-danger:disabled{color:#fff;background-color:#C71C22;border-color:#C71C22}.btn-danger:not(:disabled):not(.disabled):active,.btn-danger:not(:disabled):not(.disabled).active,.show>.btn-danger.dropdown-toggle{color:#fff;background-color:#9a161a;border-color:#8f1418}.btn-danger:not(:disabled):not(.disabled):active:focus,.btn-danger:not(:disabled):not(.disabled).active:focus,.show>.btn-danger.dropdown-toggle:focus{-webkit-box-shadow:0 0 0 0.2rem rgba(199,28,34,0.5);box-shadow:0 0 0 0.2rem rgba(199,28,34,0.5)}.btn-light{color:#212529;background-color:#f8f9fa;border-color:#f8f9fa}.btn-light:hover{color:#212529;background-color:#e2e6ea;border-color:#dae0e5}.btn-light:focus,.btn-light.focus{-webkit-box-shadow:0 0 0 0.2rem rgba(248,249,250,0.5);box-shadow:0 0 0 0.2rem rgba(248,249,250,0.5)}.btn-light.disabled,.btn-light:disabled{color:#212529;background-color:#f8f9fa;border-color:#f8f9fa}.btn-light:not(:disabled):not(.disabled):active,.btn-light:not(:disabled):not(.disabled).active,.show>.btn-light.dropdown-toggle{color:#212529;background-color:#dae0e5;border-color:#d3d9df}.btn-light:not(:disabled):not(.disabled):active:focus,.btn-light:not(:disabled):not(.disabled).active:focus,.show>.btn-light.dropdown-toggle:focus{-webkit-box-shadow:0 0 0 0.2rem rgba(248,249,250,0.5);box-shadow:0 0 0 0.2rem rgba(248,249,250,0.5)}.btn-dark{color:#fff;background-color:#343a40;border-color:#343a40}.btn-dark:hover{color:#fff;background-color:#23272b;border-color:#1d2124}.btn-dark:focus,.btn-dark.focus{-webkit-box-shadow:0 0 0 0.2rem rgba(52,58,64,0.5);box-shadow:0 0 0 0.2rem rgba(52,58,64,0.5)}.btn-dark.disabled,.btn-dark:disabled{color:#fff;background-color:#343a40;border-color:#343a40}.btn-dark:not(:disabled):not(.disabled):active,.btn-dark:not(:disabled):not(.disabled).active,.show>.btn-dark.dropdown-toggle{color:#fff;background-color:#1d2124;border-color:#171a1d}.btn-dark:not(:disabled):not(.disabled):active:focus,.btn-dark:not(:disabled):not(.disabled).active:focus,.show>.btn-dark.dropdown-toggle:focus{-webkit-box-shadow:0 0 0 0.2rem rgba(52,58,64,0.5);box-shadow:0 0 0 0.2rem rgba(52,58,64,0.5)}.btn-outline-primary{color:#2FA4E7;background-color:transparent;background-image:none;border-color:#2FA4E7}.btn-outline-primary:hover{color:#fff;background-color:#2FA4E7;border-color:#2FA4E7}.btn-outline-primary:focus,.btn-outline-primary.focus{-webkit-box-shadow:0 0 0 0.2rem rgba(47,164,231,0.5);box-shadow:0 0 0 0.2rem rgba(47,164,231,0.5)}.btn-outline-primary.disabled,.btn-outline-primary:disabled{color:#2FA4E7;background-color:transparent}.btn-outline-primary:not(:disabled):not(.disabled):active,.btn-outline-primary:not(:disabled):not(.disabled).active,.show>.btn-outline-primary.dropdown-toggle{color:#fff;background-color:#2FA4E7;border-color:#2FA4E7}.btn-outline-primary:not(:disabled):not(.disabled):active:focus,.btn-outline-primary:not(:disabled):not(.disabled).active:focus,.show>.btn-outline-primary.dropdown-toggle:focus{-webkit-box-shadow:0 0 0 0.2rem rgba(47,164,231,0.5);box-shadow:0 0 0 0.2rem rgba(47,164,231,0.5)}.btn-outline-secondary{color:#e9ecef;background-color:transparent;background-image:none;border-color:#e9ecef}.btn-outline-secondary:hover{color:#212529;background-color:#e9ecef;border-color:#e9ecef}.btn-outline-secondary:focus,.btn-outline-secondary.focus{-webkit-box-shadow:0 0 0 0.2rem rgba(233,236,239,0.5);box-shadow:0 0 0 0.2rem rgba(233,236,239,0.5)}.btn-outline-secondary.disabled,.btn-outline-secondary:disabled{color:#e9ecef;background-color:transparent}.btn-outline-secondary:not(:disabled):not(.disabled):active,.btn-outline-secondary:not(:disabled):not(.disabled).active,.show>.btn-outline-secondary.dropdown-toggle{color:#212529;background-color:#e9ecef;border-color:#e9ecef}.btn-outline-secondary:not(:disabled):not(.disabled):active:focus,.btn-outline-secondary:not(:disabled):not(.disabled).active:focus,.show>.btn-outline-secondary.dropdown-toggle:focus{-webkit-box-shadow:0 0 0 0.2rem rgba(233,236,239,0.5);box-shadow:0 0 0 0.2rem rgba(233,236,239,0.5)}.btn-outline-success{color:#73A839;background-color:transparent;background-image:none;border-color:#73A839}.btn-outline-success:hover{color:#fff;background-color:#73A839;border-color:#73A839}.btn-outline-success:focus,.btn-outline-success.focus{-webkit-box-shadow:0 0 0 0.2rem rgba(115,168,57,0.5);box-shadow:0 0 0 0.2rem rgba(115,168,57,0.5)}.btn-outline-success.disabled,.btn-outline-success:disabled{color:#73A839;background-color:transparent}.btn-outline-success:not(:disabled):not(.disabled):active,.btn-outline-success:not(:disabled):not(.disabled).active,.show>.btn-outline-success.dropdown-toggle{color:#fff;background-color:#73A839;border-color:#73A839}.btn-outline-success:not(:disabled):not(.disabled):active:focus,.btn-outline-success:not(:disabled):not(.disabled).active:focus,.show>.btn-outline-success.dropdown-toggle:focus{-webkit-box-shadow:0 0 0 0.2rem rgba(115,168,57,0.5);box-shadow:0 0 0 0.2rem rgba(115,168,57,0.5)}.btn-outline-info{color:#033C73;background-color:transparent;background-image:none;border-color:#033C73}.btn-outline-info:hover{color:#fff;background-color:#033C73;border-color:#033C73}.btn-outline-info:focus,.btn-outline-info.focus{-webkit-box-shadow:0 0 0 0.2rem rgba(3,60,115,0.5);box-shadow:0 0 0 0.2rem rgba(3,60,115,0.5)}.btn-outline-info.disabled,.btn-outline-info:disabled{color:#033C73;background-color:transparent}.btn-outline-info:not(:disabled):not(.disabled):active,.btn-outline-info:not(:disabled):not(.disabled).active,.show>.btn-outline-info.dropdown-toggle{color:#fff;background-color:#033C73;border-color:#033C73}.btn-outline-info:not(:disabled):not(.disabled):active:focus,.btn-outline-info:not(:disabled):not(.disabled).active:focus,.show>.btn-outline-info.dropdown-toggle:focus{-webkit-box-shadow:0 0 0 0.2rem rgba(3,60,115,0.5);box-shadow:0 0 0 0.2rem rgba(3,60,115,0.5)}.btn-outline-warning{color:#DD5600;background-color:transparent;background-image:none;border-color:#DD5600}.btn-outline-warning:hover{color:#fff;background-color:#DD5600;border-color:#DD5600}.btn-outline-warning:focus,.btn-outline-warning.focus{-webkit-box-shadow:0 0 0 0.2rem rgba(221,86,0,0.5);box-shadow:0 0 0 0.2rem rgba(221,86,0,0.5)}.btn-outline-warning.disabled,.btn-outline-warning:disabled{color:#DD5600;background-color:transparent}.btn-outline-warning:not(:disabled):not(.disabled):active,.btn-outline-warning:not(:disabled):not(.disabled).active,.show>.btn-outline-warning.dropdown-toggle{color:#fff;background-color:#DD5600;border-color:#DD5600}.btn-outline-warning:not(:disabled):not(.disabled):active:focus,.btn-outline-warning:not(:disabled):not(.disabled).active:focus,.show>.btn-outline-warning.dropdown-toggle:focus{-webkit-box-shadow:0 0 0 0.2rem rgba(221,86,0,0.5);box-shadow:0 0 0 0.2rem rgba(221,86,0,0.5)}.btn-outline-danger{color:#C71C22;background-color:transparent;background-image:none;border-color:#C71C22}.btn-outline-danger:hover{color:#fff;background-color:#C71C22;border-color:#C71C22}.btn-outline-danger:focus,.btn-outline-danger.focus{-webkit-box-shadow:0 0 0 0.2rem rgba(199,28,34,0.5);box-shadow:0 0 0 0.2rem rgba(199,28,34,0.5)}.btn-outline-danger.disabled,.btn-outline-danger:disabled{color:#C71C22;background-color:transparent}.btn-outline-danger:not(:disabled):not(.disabled):active,.btn-outline-danger:not(:disabled):not(.disabled).active,.show>.btn-outline-danger.dropdown-toggle{color:#fff;background-color:#C71C22;border-color:#C71C22}.btn-outline-danger:not(:disabled):not(.disabled):active:focus,.btn-outline-danger:not(:disabled):not(.disabled).active:focus,.show>.btn-outline-danger.dropdown-toggle:focus{-webkit-box-shadow:0 0 0 0.2rem rgba(199,28,34,0.5);box-shadow:0 0 0 0.2rem rgba(199,28,34,0.5)}.btn-outline-light{color:#f8f9fa;background-color:transparent;background-image:none;border-color:#f8f9fa}.btn-outline-light:hover{color:#212529;background-color:#f8f9fa;border-color:#f8f9fa}.btn-outline-light:focus,.btn-outline-light.focus{-webkit-box-shadow:0 0 0 0.2rem rgba(248,249,250,0.5);box-shadow:0 0 0 0.2rem rgba(248,249,250,0.5)}.btn-outline-light.disabled,.btn-outline-light:disabled{color:#f8f9fa;background-color:transparent}.btn-outline-light:not(:disabled):not(.disabled):active,.btn-outline-light:not(:disabled):not(.disabled).active,.show>.btn-outline-light.dropdown-toggle{color:#212529;background-color:#f8f9fa;border-color:#f8f9fa}.btn-outline-light:not(:disabled):not(.disabled):active:focus,.btn-outline-light:not(:disabled):not(.disabled).active:focus,.show>.btn-outline-light.dropdown-toggle:focus{-webkit-box-shadow:0 0 0 0.2rem rgba(248,249,250,0.5);box-shadow:0 0 0 0.2rem rgba(248,249,250,0.5)}.btn-outline-dark{color:#343a40;background-color:transparent;background-image:none;border-color:#343a40}.btn-outline-dark:hover{color:#fff;background-color:#343a40;border-color:#343a40}.btn-outline-dark:focus,.btn-outline-dark.focus{-webkit-box-shadow:0 0 0 0.2rem rgba(52,58,64,0.5);box-shadow:0 0 0 0.2rem rgba(52,58,64,0.5)}.btn-outline-dark.disabled,.btn-outline-dark:disabled{color:#343a40;background-color:transparent}.btn-outline-dark:not(:disabled):not(.disabled):active,.btn-outline-dark:not(:disabled):not(.disabled).active,.show>.btn-outline-dark.dropdown-toggle{color:#fff;background-color:#343a40;border-color:#343a40}.btn-outline-dark:not(:disabled):not(.disabled):active:focus,.btn-outline-dark:not(:disabled):not(.disabled).active:focus,.show>.btn-outline-dark.dropdown-toggle:focus{-webkit-box-shadow:0 0 0 0.2rem rgba(52,58,64,0.5);box-shadow:0 0 0 0.2rem rgba(52,58,64,0.5)}.btn-link{font-weight:400;color:#2FA4E7;background-color:transparent}.btn-link:hover{color:#157ab5;text-decoration:underline;background-color:transparent;border-color:transparent}.btn-link:focus,.btn-link.focus{text-decoration:underline;border-color:transparent;-webkit-box-shadow:none;box-shadow:none}.btn-link:disabled,.btn-link.disabled{color:#868e96;pointer-events:none}.btn-lg,.btn-group-lg>.btn{padding:0.5rem 1rem;font-size:1.25rem;line-height:1.5;border-radius:0.3rem}.btn-sm,.btn-group-sm>.btn{padding:0.25rem 0.5rem;font-size:0.875rem;line-height:1.5;border-radius:0.2rem}.btn-block{display:block;width:100%}.btn-block+.btn-block{margin-top:0.5rem}input[type="submit"].btn-block,input[type="reset"].btn-block,input[type="button"].btn-block{width:100%}.fade{-webkit-transition:opacity 0.15s linear;transition:opacity 0.15s linear}@media screen and (prefers-reduced-motion: reduce){.fade{-webkit-transition:none;transition:none}}.fade:not(.show){opacity:0}.collapse:not(.show){display:none}.collapsing{position:relative;height:0;overflow:hidden;-webkit-transition:height 0.35s ease;transition:height 0.35s ease}@media screen and (prefers-reduced-motion: reduce){.collapsing{-webkit-transition:none;transition:none}}.dropup,.dropright,.dropdown,.dropleft{position:relative}.dropdown-toggle::after{display:inline-block;width:0;height:0;margin-left:0.255em;vertical-align:0.255em;content:"";border-top:0.3em solid;border-right:0.3em solid transparent;border-bottom:0;border-left:0.3em solid transparent}.dropdown-toggle:empty::after{margin-left:0}.dropdown-menu{position:absolute;top:100%;left:0;z-index:1000;display:none;float:left;min-width:10rem;padding:0.5rem 0;margin:0.125rem 0 0;font-size:1rem;color:#495057;text-align:left;list-style:none;background-color:#fff;background-clip:padding-box;border:1px solid rgba(0,0,0,0.15);border-radius:0.25rem}.dropdown-menu-right{right:0;left:auto}.dropup .dropdown-menu{top:auto;bottom:100%;margin-top:0;margin-bottom:0.125rem}.dropup .dropdown-toggle::after{display:inline-block;width:0;height:0;margin-left:0.255em;vertical-align:0.255em;content:"";border-top:0;border-right:0.3em solid transparent;border-bottom:0.3em solid;border-left:0.3em solid transparent}.dropup .dropdown-toggle:empty::after{margin-left:0}.dropright .dropdown-menu{top:0;right:auto;left:100%;margin-top:0;margin-left:0.125rem}.dropright .dropdown-toggle::after{display:inline-block;width:0;height:0;margin-left:0.255em;vertical-align:0.255em;content:"";border-top:0.3em solid transparent;border-right:0;border-bottom:0.3em solid transparent;border-left:0.3em solid}.dropright .dropdown-toggle:empty::after{margin-left:0}.dropright .dropdown-toggle::after{vertical-align:0}.dropleft .dropdown-menu{top:0;right:100%;left:auto;margin-top:0;margin-right:0.125rem}.dropleft .dropdown-toggle::after{display:inline-block;width:0;height:0;margin-left:0.255em;vertical-align:0.255em;content:""}.dropleft .dropdown-toggle::after{display:none}.dropleft .dropdown-toggle::before{display:inline-block;width:0;height:0;margin-right:0.255em;vertical-align:0.255em;content:"";border-top:0.3em solid transparent;border-right:0.3em solid;border-bottom:0.3em solid transparent}.dropleft .dropdown-toggle:empty::after{margin-left:0}.dropleft .dropdown-toggle::before{vertical-align:0}.dropdown-menu[x-placement^="top"],.dropdown-menu[x-placement^="right"],.dropdown-menu[x-placement^="bottom"],.dropdown-menu[x-placement^="left"]{right:auto;bottom:auto}.dropdown-divider{height:0;margin:0.5rem 0;overflow:hidden;border-top:1px solid #e9ecef}.dropdown-item{display:block;width:100%;padding:0.25rem 1.5rem;clear:both;font-weight:400;color:#495057;text-align:inherit;white-space:nowrap;background-color:transparent;border:0}.dropdown-item:hover,.dropdown-item:focus{color:#fff;text-decoration:none;background-color:#2FA4E7}.dropdown-item.active,.dropdown-item:active{color:#fff;text-decoration:none;background-color:#2FA4E7}.dropdown-item.disabled,.dropdown-item:disabled{color:#868e96;background-color:transparent}.dropdown-menu.show{display:block}.dropdown-header{display:block;padding:0.5rem 1.5rem;margin-bottom:0;font-size:0.875rem;color:#868e96;white-space:nowrap}.dropdown-item-text{display:block;padding:0.25rem 1.5rem;color:#495057}.btn-group,.btn-group-vertical{position:relative;display:-webkit-inline-box;display:-ms-inline-flexbox;display:inline-flex;vertical-align:middle}.btn-group>.btn,.btn-group-vertical>.btn{position:relative;-webkit-box-flex:0;-ms-flex:0 1 auto;flex:0 1 auto}.btn-group>.btn:hover,.btn-group-vertical>.btn:hover{z-index:1}.btn-group>.btn:focus,.btn-group>.btn:active,.btn-group>.btn.active,.btn-group-vertical>.btn:focus,.btn-group-vertical>.btn:active,.btn-group-vertical>.btn.active{z-index:1}.btn-group .btn+.btn,.btn-group .btn+.btn-group,.btn-group .btn-group+.btn,.btn-group .btn-group+.btn-group,.btn-group-vertical .btn+.btn,.btn-group-vertical .btn+.btn-group,.btn-group-vertical .btn-group+.btn,.btn-group-vertical .btn-group+.btn-group{margin-left:-1px}.btn-toolbar{display:-webkit-box;display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;-webkit-box-pack:start;-ms-flex-pack:start;justify-content:flex-start}.btn-toolbar .input-group{width:auto}.btn-group>.btn:first-child{margin-left:0}.btn-group>.btn:not(:last-child):not(.dropdown-toggle),.btn-group>.btn-group:not(:last-child)>.btn{border-top-right-radius:0;border-bottom-right-radius:0}.btn-group>.btn:not(:first-child),.btn-group>.btn-group:not(:first-child)>.btn{border-top-left-radius:0;border-bottom-left-radius:0}.dropdown-toggle-split{padding-right:0.5625rem;padding-left:0.5625rem}.dropdown-toggle-split::after,.dropup .dropdown-toggle-split::after,.dropright .dropdown-toggle-split::after{margin-left:0}.dropleft .dropdown-toggle-split::before{margin-right:0}.btn-sm+.dropdown-toggle-split,.btn-group-sm>.btn+.dropdown-toggle-split{padding-right:0.375rem;padding-left:0.375rem}.btn-lg+.dropdown-toggle-split,.btn-group-lg>.btn+.dropdown-toggle-split{padding-right:0.75rem;padding-left:0.75rem}.btn-group-vertical{-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column;-webkit-box-align:start;-ms-flex-align:start;align-items:flex-start;-webkit-box-pack:center;-ms-flex-pack:center;justify-content:center}.btn-group-vertical .btn,.btn-group-vertical .btn-group{width:100%}.btn-group-vertical>.btn+.btn,.btn-group-vertical>.btn+.btn-group,.btn-group-vertical>.btn-group+.btn,.btn-group-vertical>.btn-group+.btn-group{margin-top:-1px;margin-left:0}.btn-group-vertical>.btn:not(:last-child):not(.dropdown-toggle),.btn-group-vertical>.btn-group:not(:last-child)>.btn{border-bottom-right-radius:0;border-bottom-left-radius:0}.btn-group-vertical>.btn:not(:first-child),.btn-group-vertical>.btn-group:not(:first-child)>.btn{border-top-left-radius:0;border-top-right-radius:0}.btn-group-toggle>.btn,.btn-group-toggle>.btn-group>.btn{margin-bottom:0}.btn-group-toggle>.btn input[type="radio"],.btn-group-toggle>.btn input[type="checkbox"],.btn-group-toggle>.btn-group>.btn input[type="radio"],.btn-group-toggle>.btn-group>.btn input[type="checkbox"]{position:absolute;clip:rect(0, 0, 0, 0);pointer-events:none}.input-group{position:relative;display:-webkit-box;display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;-webkit-box-align:stretch;-ms-flex-align:stretch;align-items:stretch;width:100%}.input-group>.form-control,.input-group>.custom-select,.input-group>.custom-file{position:relative;-webkit-box-flex:1;-ms-flex:1 1 auto;flex:1 1 auto;width:1%;margin-bottom:0}.input-group>.form-control+.form-control,.input-group>.form-control+.custom-select,.input-group>.form-control+.custom-file,.input-group>.custom-select+.form-control,.input-group>.custom-select+.custom-select,.input-group>.custom-select+.custom-file,.input-group>.custom-file+.form-control,.input-group>.custom-file+.custom-select,.input-group>.custom-file+.custom-file{margin-left:-1px}.input-group>.form-control:focus,.input-group>.custom-select:focus,.input-group>.custom-file .custom-file-input:focus ~ .custom-file-label{z-index:3}.input-group>.custom-file .custom-file-input:focus{z-index:4}.input-group>.form-control:not(:last-child),.input-group>.custom-select:not(:last-child){border-top-right-radius:0;border-bottom-right-radius:0}.input-group>.form-control:not(:first-child),.input-group>.custom-select:not(:first-child){border-top-left-radius:0;border-bottom-left-radius:0}.input-group>.custom-file{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-align:center;-ms-flex-align:center;align-items:center}.input-group>.custom-file:not(:last-child) .custom-file-label,.input-group>.custom-file:not(:last-child) .custom-file-label::after{border-top-right-radius:0;border-bottom-right-radius:0}.input-group>.custom-file:not(:first-child) .custom-file-label{border-top-left-radius:0;border-bottom-left-radius:0}.input-group-prepend,.input-group-append{display:-webkit-box;display:-ms-flexbox;display:flex}.input-group-prepend .btn,.input-group-append .btn{position:relative;z-index:2}.input-group-prepend .btn+.btn,.input-group-prepend .btn+.input-group-text,.input-group-prepend .input-group-text+.input-group-text,.input-group-prepend .input-group-text+.btn,.input-group-append .btn+.btn,.input-group-append .btn+.input-group-text,.input-group-append .input-group-text+.input-group-text,.input-group-append .input-group-text+.btn{margin-left:-1px}.input-group-prepend{margin-right:-1px}.input-group-append{margin-left:-1px}.input-group-text{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-align:center;-ms-flex-align:center;align-items:center;padding:0.375rem 0.75rem;margin-bottom:0;font-size:1rem;font-weight:400;line-height:1.5;color:#495057;text-align:center;white-space:nowrap;background-color:#e9ecef;border:1px solid #ced4da;border-radius:0.25rem}.input-group-text input[type="radio"],.input-group-text input[type="checkbox"]{margin-top:0}.input-group-lg>.form-control,.input-group-lg>.input-group-prepend>.input-group-text,.input-group-lg>.input-group-append>.input-group-text,.input-group-lg>.input-group-prepend>.btn,.input-group-lg>.input-group-append>.btn{height:calc(2.875rem + 2px);padding:0.5rem 1rem;font-size:1.25rem;line-height:1.5;border-radius:0.3rem}.input-group-sm>.form-control,.input-group-sm>.input-group-prepend>.input-group-text,.input-group-sm>.input-group-append>.input-group-text,.input-group-sm>.input-group-prepend>.btn,.input-group-sm>.input-group-append>.btn{height:calc(1.8125rem + 2px);padding:0.25rem 0.5rem;font-size:0.875rem;line-height:1.5;border-radius:0.2rem}.input-group>.input-group-prepend>.btn,.input-group>.input-group-prepend>.input-group-text,.input-group>.input-group-append:not(:last-child)>.btn,.input-group>.input-group-append:not(:last-child)>.input-group-text,.input-group>.input-group-append:last-child>.btn:not(:last-child):not(.dropdown-toggle),.input-group>.input-group-append:last-child>.input-group-text:not(:last-child){border-top-right-radius:0;border-bottom-right-radius:0}.input-group>.input-group-append>.btn,.input-group>.input-group-append>.input-group-text,.input-group>.input-group-prepend:not(:first-child)>.btn,.input-group>.input-group-prepend:not(:first-child)>.input-group-text,.input-group>.input-group-prepend:first-child>.btn:not(:first-child),.input-group>.input-group-prepend:first-child>.input-group-text:not(:first-child){border-top-left-radius:0;border-bottom-left-radius:0}.custom-control{position:relative;display:block;min-height:1.5rem;padding-left:1.5rem}.custom-control-inline{display:-webkit-inline-box;display:-ms-inline-flexbox;display:inline-flex;margin-right:1rem}.custom-control-input{position:absolute;z-index:-1;opacity:0}.custom-control-input:checked ~ .custom-control-label::before{color:#fff;background-color:#2FA4E7}.custom-control-input:focus ~ .custom-control-label::before{-webkit-box-shadow:0 0 0 1px #fff,0 0 0 0.2rem rgba(47,164,231,0.25);box-shadow:0 0 0 1px #fff,0 0 0 0.2rem rgba(47,164,231,0.25)}.custom-control-input:active ~ .custom-control-label::before{color:#fff;background-color:#cfeaf9}.custom-control-input:disabled ~ .custom-control-label{color:#868e96}.custom-control-input:disabled ~ .custom-control-label::before{background-color:#e9ecef}.custom-control-label{position:relative;margin-bottom:0}.custom-control-label::before{position:absolute;top:0.25rem;left:-1.5rem;display:block;width:1rem;height:1rem;pointer-events:none;content:"";-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;background-color:#dee2e6}.custom-control-label::after{position:absolute;top:0.25rem;left:-1.5rem;display:block;width:1rem;height:1rem;content:"";background-repeat:no-repeat;background-position:center center;background-size:50% 50%}.custom-checkbox .custom-control-label::before{border-radius:0.25rem}.custom-checkbox .custom-control-input:checked ~ .custom-control-label::before{background-color:#2FA4E7}.custom-checkbox .custom-control-input:checked ~ .custom-control-label::after{background-image:url("data:image/svg+xml;charset=utf8,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 8 8'%3E%3Cpath fill='%23fff' d='M6.564.75l-3.59 3.612-1.538-1.55L0 4.26 2.974 7.25 8 2.193z'/%3E%3C/svg%3E")}.custom-checkbox .custom-control-input:indeterminate ~ .custom-control-label::before{background-color:#2FA4E7}.custom-checkbox .custom-control-input:indeterminate ~ .custom-control-label::after{background-image:url("data:image/svg+xml;charset=utf8,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 4 4'%3E%3Cpath stroke='%23fff' d='M0 2h4'/%3E%3C/svg%3E")}.custom-checkbox .custom-control-input:disabled:checked ~ .custom-control-label::before{background-color:rgba(47,164,231,0.5)}.custom-checkbox .custom-control-input:disabled:indeterminate ~ .custom-control-label::before{background-color:rgba(47,164,231,0.5)}.custom-radio .custom-control-label::before{border-radius:50%}.custom-radio .custom-control-input:checked ~ .custom-control-label::before{background-color:#2FA4E7}.custom-radio .custom-control-input:checked ~ .custom-control-label::after{background-image:url("data:image/svg+xml;charset=utf8,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='-4 -4 8 8'%3E%3Ccircle r='3' fill='%23fff'/%3E%3C/svg%3E")}.custom-radio .custom-control-input:disabled:checked ~ .custom-control-label::before{background-color:rgba(47,164,231,0.5)}.custom-select{display:inline-block;width:100%;height:calc(2.25rem + 2px);padding:0.375rem 1.75rem 0.375rem 0.75rem;line-height:1.5;color:#495057;vertical-align:middle;background:#fff url("data:image/svg+xml;charset=utf8,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 4 5'%3E%3Cpath fill='%23343a40' d='M2 0L0 2h4zm0 5L0 3h4z'/%3E%3C/svg%3E") no-repeat right 0.75rem center;background-size:8px 10px;border:1px solid #ced4da;border-radius:0.25rem;-webkit-appearance:none;-moz-appearance:none;appearance:none}.custom-select:focus{border-color:#a1d6f4;outline:0;-webkit-box-shadow:0 0 0 0.2rem rgba(161,214,244,0.5);box-shadow:0 0 0 0.2rem rgba(161,214,244,0.5)}.custom-select:focus::-ms-value{color:#495057;background-color:#fff}.custom-select[multiple],.custom-select[size]:not([size="1"]){height:auto;padding-right:0.75rem;background-image:none}.custom-select:disabled{color:#868e96;background-color:#e9ecef}.custom-select::-ms-expand{opacity:0}.custom-select-sm{height:calc(1.8125rem + 2px);padding-top:0.375rem;padding-bottom:0.375rem;font-size:75%}.custom-select-lg{height:calc(2.875rem + 2px);padding-top:0.375rem;padding-bottom:0.375rem;font-size:125%}.custom-file{position:relative;display:inline-block;width:100%;height:calc(2.25rem + 2px);margin-bottom:0}.custom-file-input{position:relative;z-index:2;width:100%;height:calc(2.25rem + 2px);margin:0;opacity:0}.custom-file-input:focus ~ .custom-file-label{border-color:#a1d6f4;-webkit-box-shadow:0 0 0 0.2rem rgba(47,164,231,0.25);box-shadow:0 0 0 0.2rem rgba(47,164,231,0.25)}.custom-file-input:focus ~ .custom-file-label::after{border-color:#a1d6f4}.custom-file-input:disabled ~ .custom-file-label{background-color:#e9ecef}.custom-file-input:lang(en) ~ .custom-file-label::after{content:"Browse"}.custom-file-label{position:absolute;top:0;right:0;left:0;z-index:1;height:calc(2.25rem + 2px);padding:0.375rem 0.75rem;line-height:1.5;color:#495057;background-color:#fff;border:1px solid #ced4da;border-radius:0.25rem}.custom-file-label::after{position:absolute;top:0;right:0;bottom:0;z-index:3;display:block;height:2.25rem;padding:0.375rem 0.75rem;line-height:1.5;color:#495057;content:"Browse";background-color:#e9ecef;border-left:1px solid #ced4da;border-radius:0 0.25rem 0.25rem 0}.custom-range{width:100%;padding-left:0;background-color:transparent;-webkit-appearance:none;-moz-appearance:none;appearance:none}.custom-range:focus{outline:none}.custom-range:focus::-webkit-slider-thumb{-webkit-box-shadow:0 0 0 1px #fff,0 0 0 0.2rem rgba(47,164,231,0.25);box-shadow:0 0 0 1px #fff,0 0 0 0.2rem rgba(47,164,231,0.25)}.custom-range:focus::-moz-range-thumb{box-shadow:0 0 0 1px #fff,0 0 0 0.2rem rgba(47,164,231,0.25)}.custom-range:focus::-ms-thumb{box-shadow:0 0 0 1px #fff,0 0 0 0.2rem rgba(47,164,231,0.25)}.custom-range::-moz-focus-outer{border:0}.custom-range::-webkit-slider-thumb{width:1rem;height:1rem;margin-top:-0.25rem;background-color:#2FA4E7;border:0;border-radius:1rem;-webkit-transition:background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, -webkit-box-shadow 0.15s ease-in-out;transition:background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, -webkit-box-shadow 0.15s ease-in-out;transition:background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out;transition:background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out, -webkit-box-shadow 0.15s ease-in-out;-webkit-appearance:none;appearance:none}@media screen and (prefers-reduced-motion: reduce){.custom-range::-webkit-slider-thumb{-webkit-transition:none;transition:none}}.custom-range::-webkit-slider-thumb:active{background-color:#cfeaf9}.custom-range::-webkit-slider-runnable-track{width:100%;height:0.5rem;color:transparent;cursor:pointer;background-color:#dee2e6;border-color:transparent;border-radius:1rem}.custom-range::-moz-range-thumb{width:1rem;height:1rem;background-color:#2FA4E7;border:0;border-radius:1rem;-webkit-transition:background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, -webkit-box-shadow 0.15s ease-in-out;transition:background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, -webkit-box-shadow 0.15s ease-in-out;transition:background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out;transition:background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out, -webkit-box-shadow 0.15s ease-in-out;-moz-appearance:none;appearance:none}@media screen and (prefers-reduced-motion: reduce){.custom-range::-moz-range-thumb{-webkit-transition:none;transition:none}}.custom-range::-moz-range-thumb:active{background-color:#cfeaf9}.custom-range::-moz-range-track{width:100%;height:0.5rem;color:transparent;cursor:pointer;background-color:#dee2e6;border-color:transparent;border-radius:1rem}.custom-range::-ms-thumb{width:1rem;height:1rem;margin-top:0;margin-right:0.2rem;margin-left:0.2rem;background-color:#2FA4E7;border:0;border-radius:1rem;-webkit-transition:background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, -webkit-box-shadow 0.15s ease-in-out;transition:background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, -webkit-box-shadow 0.15s ease-in-out;transition:background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out;transition:background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out, -webkit-box-shadow 0.15s ease-in-out;appearance:none}@media screen and (prefers-reduced-motion: reduce){.custom-range::-ms-thumb{-webkit-transition:none;transition:none}}.custom-range::-ms-thumb:active{background-color:#cfeaf9}.custom-range::-ms-track{width:100%;height:0.5rem;color:transparent;cursor:pointer;background-color:transparent;border-color:transparent;border-width:0.5rem}.custom-range::-ms-fill-lower{background-color:#dee2e6;border-radius:1rem}.custom-range::-ms-fill-upper{margin-right:15px;background-color:#dee2e6;border-radius:1rem}.custom-control-label::before,.custom-file-label,.custom-select{-webkit-transition:background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, -webkit-box-shadow 0.15s ease-in-out;transition:background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, -webkit-box-shadow 0.15s ease-in-out;transition:background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out;transition:background-color 0.15s ease-in-out, border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out, -webkit-box-shadow 0.15s ease-in-out}@media screen and (prefers-reduced-motion: reduce){.custom-control-label::before,.custom-file-label,.custom-select{-webkit-transition:none;transition:none}}.nav{display:-webkit-box;display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;padding-left:0;margin-bottom:0;list-style:none}.nav-link{display:block;padding:0.5rem 1rem}.nav-link:hover,.nav-link:focus{text-decoration:none}.nav-link.disabled{color:#868e96}.nav-tabs{border-bottom:1px solid #dee2e6}.nav-tabs .nav-item{margin-bottom:-1px}.nav-tabs .nav-link{border:1px solid transparent;border-top-left-radius:0.25rem;border-top-right-radius:0.25rem}.nav-tabs .nav-link:hover,.nav-tabs .nav-link:focus{border-color:#e9ecef #e9ecef #dee2e6}.nav-tabs .nav-link.disabled{color:#868e96;background-color:transparent;border-color:transparent}.nav-tabs .nav-link.active,.nav-tabs .nav-item.show .nav-link{color:#495057;background-color:#fff;border-color:#dee2e6 #dee2e6 #fff}.nav-tabs .dropdown-menu{margin-top:-1px;border-top-left-radius:0;border-top-right-radius:0}.nav-pills .nav-link{border-radius:0.25rem}.nav-pills .nav-link.active,.nav-pills .show>.nav-link{color:#fff;background-color:#2FA4E7}.nav-fill .nav-item{-webkit-box-flex:1;-ms-flex:1 1 auto;flex:1 1 auto;text-align:center}.nav-justified .nav-item{-ms-flex-preferred-size:0;flex-basis:0;-webkit-box-flex:1;-ms-flex-positive:1;flex-grow:1;text-align:center}.tab-content>.tab-pane{display:none}.tab-content>.active{display:block}.navbar{position:relative;display:-webkit-box;display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;-webkit-box-align:center;-ms-flex-align:center;align-items:center;-webkit-box-pack:justify;-ms-flex-pack:justify;justify-content:space-between;padding:0.5rem 1rem}.navbar>.container,.navbar>.container-fluid{display:-webkit-box;display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;-webkit-box-align:center;-ms-flex-align:center;align-items:center;-webkit-box-pack:justify;-ms-flex-pack:justify;justify-content:space-between}.navbar-brand{display:inline-block;padding-top:0.3125rem;padding-bottom:0.3125rem;margin-right:1rem;font-size:1.25rem;line-height:inherit;white-space:nowrap}.navbar-brand:hover,.navbar-brand:focus{text-decoration:none}.navbar-nav{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column;padding-left:0;margin-bottom:0;list-style:none}.navbar-nav .nav-link{padding-right:0;padding-left:0}.navbar-nav .dropdown-menu{position:static;float:none}.navbar-text{display:inline-block;padding-top:0.5rem;padding-bottom:0.5rem}.navbar-collapse{-ms-flex-preferred-size:100%;flex-basis:100%;-webkit-box-flex:1;-ms-flex-positive:1;flex-grow:1;-webkit-box-align:center;-ms-flex-align:center;align-items:center}.navbar-toggler{padding:0.25rem 0.75rem;font-size:1.25rem;line-height:1;background-color:transparent;border:1px solid transparent;border-radius:0.25rem}.navbar-toggler:hover,.navbar-toggler:focus{text-decoration:none}.navbar-toggler:not(:disabled):not(.disabled){cursor:pointer}.navbar-toggler-icon{display:inline-block;width:1.5em;height:1.5em;vertical-align:middle;content:"";background:no-repeat center center;background-size:100% 100%}@media (max-width: 575.98px){.navbar-expand-sm>.container,.navbar-expand-sm>.container-fluid{padding-right:0;padding-left:0}}@media (min-width: 576px){.navbar-expand-sm{-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-flow:row nowrap;flex-flow:row nowrap;-webkit-box-pack:start;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand-sm .navbar-nav{-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-direction:row;flex-direction:row}.navbar-expand-sm .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-sm .navbar-nav .nav-link{padding-right:0.5rem;padding-left:0.5rem}.navbar-expand-sm>.container,.navbar-expand-sm>.container-fluid{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand-sm .navbar-collapse{display:-webkit-box !important;display:-ms-flexbox !important;display:flex !important;-ms-flex-preferred-size:auto;flex-basis:auto}.navbar-expand-sm .navbar-toggler{display:none}}@media (max-width: 767.98px){.navbar-expand-md>.container,.navbar-expand-md>.container-fluid{padding-right:0;padding-left:0}}@media (min-width: 768px){.navbar-expand-md{-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-flow:row nowrap;flex-flow:row nowrap;-webkit-box-pack:start;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand-md .navbar-nav{-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-direction:row;flex-direction:row}.navbar-expand-md .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-md .navbar-nav .nav-link{padding-right:0.5rem;padding-left:0.5rem}.navbar-expand-md>.container,.navbar-expand-md>.container-fluid{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand-md .navbar-collapse{display:-webkit-box !important;display:-ms-flexbox !important;display:flex !important;-ms-flex-preferred-size:auto;flex-basis:auto}.navbar-expand-md .navbar-toggler{display:none}}@media (max-width: 991.98px){.navbar-expand-lg>.container,.navbar-expand-lg>.container-fluid{padding-right:0;padding-left:0}}@media (min-width: 992px){.navbar-expand-lg{-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-flow:row nowrap;flex-flow:row nowrap;-webkit-box-pack:start;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand-lg .navbar-nav{-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-direction:row;flex-direction:row}.navbar-expand-lg .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-lg .navbar-nav .nav-link{padding-right:0.5rem;padding-left:0.5rem}.navbar-expand-lg>.container,.navbar-expand-lg>.container-fluid{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand-lg .navbar-collapse{display:-webkit-box !important;display:-ms-flexbox !important;display:flex !important;-ms-flex-preferred-size:auto;flex-basis:auto}.navbar-expand-lg .navbar-toggler{display:none}}@media (max-width: 1199.98px){.navbar-expand-xl>.container,.navbar-expand-xl>.container-fluid{padding-right:0;padding-left:0}}@media (min-width: 1200px){.navbar-expand-xl{-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-flow:row nowrap;flex-flow:row nowrap;-webkit-box-pack:start;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand-xl .navbar-nav{-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-direction:row;flex-direction:row}.navbar-expand-xl .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-xl .navbar-nav .nav-link{padding-right:0.5rem;padding-left:0.5rem}.navbar-expand-xl>.container,.navbar-expand-xl>.container-fluid{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand-xl .navbar-collapse{display:-webkit-box !important;display:-ms-flexbox !important;display:flex !important;-ms-flex-preferred-size:auto;flex-basis:auto}.navbar-expand-xl .navbar-toggler{display:none}}.navbar-expand{-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-flow:row nowrap;flex-flow:row nowrap;-webkit-box-pack:start;-ms-flex-pack:start;justify-content:flex-start}.navbar-expand>.container,.navbar-expand>.container-fluid{padding-right:0;padding-left:0}.navbar-expand .navbar-nav{-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-direction:row;flex-direction:row}.navbar-expand .navbar-nav .dropdown-menu{position:absolute}.navbar-expand .navbar-nav .nav-link{padding-right:0.5rem;padding-left:0.5rem}.navbar-expand>.container,.navbar-expand>.container-fluid{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.navbar-expand .navbar-collapse{display:-webkit-box !important;display:-ms-flexbox !important;display:flex !important;-ms-flex-preferred-size:auto;flex-basis:auto}.navbar-expand .navbar-toggler{display:none}.navbar-light .navbar-brand{color:rgba(0,0,0,0.9)}.navbar-light .navbar-brand:hover,.navbar-light .navbar-brand:focus{color:rgba(0,0,0,0.9)}.navbar-light .navbar-nav .nav-link{color:rgba(0,0,0,0.5)}.navbar-light .navbar-nav .nav-link:hover,.navbar-light .navbar-nav .nav-link:focus{color:rgba(0,0,0,0.7)}.navbar-light .navbar-nav .nav-link.disabled{color:rgba(0,0,0,0.3)}.navbar-light .navbar-nav .show>.nav-link,.navbar-light .navbar-nav .active>.nav-link,.navbar-light .navbar-nav .nav-link.show,.navbar-light .navbar-nav .nav-link.active{color:rgba(0,0,0,0.9)}.navbar-light .navbar-toggler{color:rgba(0,0,0,0.5);border-color:rgba(0,0,0,0.1)}.navbar-light .navbar-toggler-icon{background-image:url("data:image/svg+xml;charset=utf8,%3Csvg viewBox='0 0 30 30' xmlns='http://www.w3.org/2000/svg'%3E%3Cpath stroke='rgba(0, 0, 0, 0.5)' stroke-width='2' stroke-linecap='round' stroke-miterlimit='10' d='M4 7h22M4 15h22M4 23h22'/%3E%3C/svg%3E")}.navbar-light .navbar-text{color:rgba(0,0,0,0.5)}.navbar-light .navbar-text a{color:rgba(0,0,0,0.9)}.navbar-light .navbar-text a:hover,.navbar-light .navbar-text a:focus{color:rgba(0,0,0,0.9)}.navbar-dark .navbar-brand{color:#fff}.navbar-dark .navbar-brand:hover,.navbar-dark .navbar-brand:focus{color:#fff}.navbar-dark .navbar-nav .nav-link{color:rgba(255,255,255,0.8)}.navbar-dark .navbar-nav .nav-link:hover,.navbar-dark .navbar-nav .nav-link:focus{color:#fff}.navbar-dark .navbar-nav .nav-link.disabled{color:rgba(255,255,255,0.25)}.navbar-dark .navbar-nav .show>.nav-link,.navbar-dark .navbar-nav .active>.nav-link,.navbar-dark .navbar-nav .nav-link.show,.navbar-dark .navbar-nav .nav-link.active{color:#fff}.navbar-dark .navbar-toggler{color:rgba(255,255,255,0.8);border-color:rgba(255,255,255,0.1)}.navbar-dark .navbar-toggler-icon{background-image:url("data:image/svg+xml;charset=utf8,%3Csvg viewBox='0 0 30 30' xmlns='http://www.w3.org/2000/svg'%3E%3Cpath stroke='rgba(255, 255, 255, 0.8)' stroke-width='2' stroke-linecap='round' stroke-miterlimit='10' d='M4 7h22M4 15h22M4 23h22'/%3E%3C/svg%3E")}.navbar-dark .navbar-text{color:rgba(255,255,255,0.8)}.navbar-dark .navbar-text a{color:#fff}.navbar-dark .navbar-text a:hover,.navbar-dark .navbar-text a:focus{color:#fff}.card{position:relative;display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column;min-width:0;word-wrap:break-word;background-color:#fff;background-clip:border-box;border:1px solid rgba(0,0,0,0.125);border-radius:0.25rem}.card>hr{margin-right:0;margin-left:0}.card>.list-group:first-child .list-group-item:first-child{border-top-left-radius:0.25rem;border-top-right-radius:0.25rem}.card>.list-group:last-child .list-group-item:last-child{border-bottom-right-radius:0.25rem;border-bottom-left-radius:0.25rem}.card-body{-webkit-box-flex:1;-ms-flex:1 1 auto;flex:1 1 auto;padding:1.25rem}.card-title{margin-bottom:0.75rem}.card-subtitle{margin-top:-0.375rem;margin-bottom:0}.card-text:last-child{margin-bottom:0}.card-link:hover{text-decoration:none}.card-link+.card-link{margin-left:1.25rem}.card-header{padding:0.75rem 1.25rem;margin-bottom:0;background-color:rgba(0,0,0,0.03);border-bottom:1px solid rgba(0,0,0,0.125)}.card-header:first-child{border-radius:calc(0.25rem - 1px) calc(0.25rem - 1px) 0 0}.card-header+.list-group .list-group-item:first-child{border-top:0}.card-footer{padding:0.75rem 1.25rem;background-color:rgba(0,0,0,0.03);border-top:1px solid rgba(0,0,0,0.125)}.card-footer:last-child{border-radius:0 0 calc(0.25rem - 1px) calc(0.25rem - 1px)}.card-header-tabs{margin-right:-0.625rem;margin-bottom:-0.75rem;margin-left:-0.625rem;border-bottom:0}.card-header-pills{margin-right:-0.625rem;margin-left:-0.625rem}.card-img-overlay{position:absolute;top:0;right:0;bottom:0;left:0;padding:1.25rem}.card-img{width:100%;border-radius:calc(0.25rem - 1px)}.card-img-top{width:100%;border-top-left-radius:calc(0.25rem - 1px);border-top-right-radius:calc(0.25rem - 1px)}.card-img-bottom{width:100%;border-bottom-right-radius:calc(0.25rem - 1px);border-bottom-left-radius:calc(0.25rem - 1px)}.card-deck{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column}.card-deck .card{margin-bottom:15px}@media (min-width: 576px){.card-deck{-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-flow:row wrap;flex-flow:row wrap;margin-right:-15px;margin-left:-15px}.card-deck .card{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-flex:1;-ms-flex:1 0 0%;flex:1 0 0%;-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column;margin-right:15px;margin-bottom:0;margin-left:15px}}.card-group{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column}.card-group>.card{margin-bottom:15px}@media (min-width: 576px){.card-group{-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-flow:row wrap;flex-flow:row wrap}.card-group>.card{-webkit-box-flex:1;-ms-flex:1 0 0%;flex:1 0 0%;margin-bottom:0}.card-group>.card+.card{margin-left:0;border-left:0}.card-group>.card:first-child{border-top-right-radius:0;border-bottom-right-radius:0}.card-group>.card:first-child .card-img-top,.card-group>.card:first-child .card-header{border-top-right-radius:0}.card-group>.card:first-child .card-img-bottom,.card-group>.card:first-child .card-footer{border-bottom-right-radius:0}.card-group>.card:last-child{border-top-left-radius:0;border-bottom-left-radius:0}.card-group>.card:last-child .card-img-top,.card-group>.card:last-child .card-header{border-top-left-radius:0}.card-group>.card:last-child .card-img-bottom,.card-group>.card:last-child .card-footer{border-bottom-left-radius:0}.card-group>.card:only-child{border-radius:0.25rem}.card-group>.card:only-child .card-img-top,.card-group>.card:only-child .card-header{border-top-left-radius:0.25rem;border-top-right-radius:0.25rem}.card-group>.card:only-child .card-img-bottom,.card-group>.card:only-child .card-footer{border-bottom-right-radius:0.25rem;border-bottom-left-radius:0.25rem}.card-group>.card:not(:first-child):not(:last-child):not(:only-child){border-radius:0}.card-group>.card:not(:first-child):not(:last-child):not(:only-child) .card-img-top,.card-group>.card:not(:first-child):not(:last-child):not(:only-child) .card-img-bottom,.card-group>.card:not(:first-child):not(:last-child):not(:only-child) .card-header,.card-group>.card:not(:first-child):not(:last-child):not(:only-child) .card-footer{border-radius:0}}.card-columns .card{margin-bottom:0.75rem}@media (min-width: 576px){.card-columns{-webkit-column-count:3;column-count:3;-webkit-column-gap:1.25rem;column-gap:1.25rem;orphans:1;widows:1}.card-columns .card{display:inline-block;width:100%}}.accordion .card:not(:first-of-type):not(:last-of-type){border-bottom:0;border-radius:0}.accordion .card:not(:first-of-type) .card-header:first-child{border-radius:0}.accordion .card:first-of-type{border-bottom:0;border-bottom-right-radius:0;border-bottom-left-radius:0}.accordion .card:last-of-type{border-top-left-radius:0;border-top-right-radius:0}.breadcrumb{display:-webkit-box;display:-ms-flexbox;display:flex;-ms-flex-wrap:wrap;flex-wrap:wrap;padding:0.75rem 1rem;margin-bottom:1rem;list-style:none;background-color:#e9ecef;border-radius:0.25rem}.breadcrumb-item+.breadcrumb-item{padding-left:0.5rem}.breadcrumb-item+.breadcrumb-item::before{display:inline-block;padding-right:0.5rem;color:#868e96;content:"/"}.breadcrumb-item+.breadcrumb-item:hover::before{text-decoration:underline}.breadcrumb-item+.breadcrumb-item:hover::before{text-decoration:none}.breadcrumb-item.active{color:#868e96}.pagination{display:-webkit-box;display:-ms-flexbox;display:flex;padding-left:0;list-style:none;border-radius:0.25rem}.page-link{position:relative;display:block;padding:0.5rem 0.75rem;margin-left:-1px;line-height:1.25;color:#2FA4E7;background-color:#fff;border:1px solid #dee2e6}.page-link:hover{z-index:2;color:#157ab5;text-decoration:none;background-color:#e9ecef;border-color:#dee2e6}.page-link:focus{z-index:2;outline:0;-webkit-box-shadow:0 0 0 0.2rem rgba(47,164,231,0.25);box-shadow:0 0 0 0.2rem rgba(47,164,231,0.25)}.page-link:not(:disabled):not(.disabled){cursor:pointer}.page-item:first-child .page-link{margin-left:0;border-top-left-radius:0.25rem;border-bottom-left-radius:0.25rem}.page-item:last-child .page-link{border-top-right-radius:0.25rem;border-bottom-right-radius:0.25rem}.page-item.active .page-link{z-index:1;color:#fff;background-color:#2FA4E7;border-color:#2FA4E7}.page-item.disabled .page-link{color:#868e96;pointer-events:none;cursor:auto;background-color:#fff;border-color:#dee2e6}.pagination-lg .page-link{padding:0.75rem 1.5rem;font-size:1.25rem;line-height:1.5}.pagination-lg .page-item:first-child .page-link{border-top-left-radius:0.3rem;border-bottom-left-radius:0.3rem}.pagination-lg .page-item:last-child .page-link{border-top-right-radius:0.3rem;border-bottom-right-radius:0.3rem}.pagination-sm .page-link{padding:0.25rem 0.5rem;font-size:0.875rem;line-height:1.5}.pagination-sm .page-item:first-child .page-link{border-top-left-radius:0.2rem;border-bottom-left-radius:0.2rem}.pagination-sm .page-item:last-child .page-link{border-top-right-radius:0.2rem;border-bottom-right-radius:0.2rem}.badge{display:inline-block;padding:0.25em 0.4em;font-size:75%;font-weight:700;line-height:1;text-align:center;white-space:nowrap;vertical-align:baseline;border-radius:0.25rem}.badge:empty{display:none}.btn .badge{position:relative;top:-1px}.badge-pill{padding-right:0.6em;padding-left:0.6em;border-radius:10rem}.badge-primary{color:#fff;background-color:#2FA4E7}.badge-primary[href]:hover,.badge-primary[href]:focus{color:#fff;text-decoration:none;background-color:#178acc}.badge-secondary{color:#212529;background-color:#e9ecef}.badge-secondary[href]:hover,.badge-secondary[href]:focus{color:#212529;text-decoration:none;background-color:#cbd3da}.badge-success{color:#fff;background-color:#73A839}.badge-success[href]:hover,.badge-success[href]:focus{color:#fff;text-decoration:none;background-color:#59822c}.badge-info{color:#fff;background-color:#033C73}.badge-info[href]:hover,.badge-info[href]:focus{color:#fff;text-decoration:none;background-color:#022241}.badge-warning{color:#fff;background-color:#DD5600}.badge-warning[href]:hover,.badge-warning[href]:focus{color:#fff;text-decoration:none;background-color:#aa4200}.badge-danger{color:#fff;background-color:#C71C22}.badge-danger[href]:hover,.badge-danger[href]:focus{color:#fff;text-decoration:none;background-color:#9a161a}.badge-light{color:#212529;background-color:#f8f9fa}.badge-light[href]:hover,.badge-light[href]:focus{color:#212529;text-decoration:none;background-color:#dae0e5}.badge-dark{color:#fff;background-color:#343a40}.badge-dark[href]:hover,.badge-dark[href]:focus{color:#fff;text-decoration:none;background-color:#1d2124}.jumbotron{padding:2rem 1rem;margin-bottom:2rem;background-color:#e9ecef;border-radius:0.3rem}@media (min-width: 576px){.jumbotron{padding:4rem 2rem}}.jumbotron-fluid{padding-right:0;padding-left:0;border-radius:0}.alert{position:relative;padding:0.75rem 1.25rem;margin-bottom:1rem;border:1px solid transparent;border-radius:0.25rem}.alert-heading{color:inherit}.alert-link{font-weight:700}.alert-dismissible{padding-right:4rem}.alert-dismissible .close{position:absolute;top:0;right:0;padding:0.75rem 1.25rem;color:inherit}.alert-primary{color:#185578;background-color:#d5edfa;border-color:#c5e6f8}.alert-primary hr{border-top-color:#aedcf5}.alert-primary .alert-link{color:#10374e}.alert-secondary{color:#797b7c;background-color:#fbfbfc;border-color:#f9fafb}.alert-secondary hr{border-top-color:#eaedf1}.alert-secondary .alert-link{color:#606162}.alert-success{color:#3c571e;background-color:#e3eed7;border-color:#d8e7c8}.alert-success hr{border-top-color:#cbdfb6}.alert-success .alert-link{color:#223111}.alert-info{color:#021f3c;background-color:#cdd8e3;border-color:#b8c8d8}.alert-info hr{border-top-color:#a8bbcf}.alert-info .alert-link{color:#00060b}.alert-warning{color:#732d00;background-color:#f8ddcc;border-color:#f5d0b8}.alert-warning hr{border-top-color:#f2c1a2}.alert-warning .alert-link{color:#401900}.alert-danger{color:#670f12;background-color:#f4d2d3;border-color:#efbfc1}.alert-danger hr{border-top-color:#eaabad}.alert-danger .alert-link{color:#3a090a}.alert-light{color:#818182;background-color:#fefefe;border-color:#fdfdfe}.alert-light hr{border-top-color:#ececf6}.alert-light .alert-link{color:#686868}.alert-dark{color:#1b1e21;background-color:#d6d8d9;border-color:#c6c8ca}.alert-dark hr{border-top-color:#b9bbbe}.alert-dark .alert-link{color:#040505}@-webkit-keyframes progress-bar-stripes{from{background-position:1rem 0}to{background-position:0 0}}@keyframes progress-bar-stripes{from{background-position:1rem 0}to{background-position:0 0}}.progress{display:-webkit-box;display:-ms-flexbox;display:flex;height:1rem;overflow:hidden;font-size:0.75rem;background-color:#e9ecef;border-radius:0.25rem}.progress-bar{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column;-webkit-box-pack:center;-ms-flex-pack:center;justify-content:center;color:#fff;text-align:center;white-space:nowrap;background-color:#2FA4E7;-webkit-transition:width 0.6s ease;transition:width 0.6s ease}@media screen and (prefers-reduced-motion: reduce){.progress-bar{-webkit-transition:none;transition:none}}.progress-bar-striped{background-image:linear-gradient(45deg, rgba(255,255,255,0.15) 25%, transparent 25%, transparent 50%, rgba(255,255,255,0.15) 50%, rgba(255,255,255,0.15) 75%, transparent 75%, transparent);background-size:1rem 1rem}.progress-bar-animated{-webkit-animation:progress-bar-stripes 1s linear infinite;animation:progress-bar-stripes 1s linear infinite}.media{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-align:start;-ms-flex-align:start;align-items:flex-start}.media-body{-webkit-box-flex:1;-ms-flex:1;flex:1}.list-group{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column;padding-left:0;margin-bottom:0}.list-group-item-action{width:100%;color:#495057;text-align:inherit}.list-group-item-action:hover,.list-group-item-action:focus{color:#495057;text-decoration:none;background-color:#f8f9fa}.list-group-item-action:active{color:#495057;background-color:#e9ecef}.list-group-item{position:relative;display:block;padding:0.75rem 1.25rem;margin-bottom:-1px;background-color:#fff;border:1px solid rgba(0,0,0,0.125)}.list-group-item:first-child{border-top-left-radius:0.25rem;border-top-right-radius:0.25rem}.list-group-item:last-child{margin-bottom:0;border-bottom-right-radius:0.25rem;border-bottom-left-radius:0.25rem}.list-group-item:hover,.list-group-item:focus{z-index:1;text-decoration:none}.list-group-item.disabled,.list-group-item:disabled{color:#868e96;background-color:#fff}.list-group-item.active{z-index:2;color:#fff;background-color:#2FA4E7;border-color:#2FA4E7}.list-group-flush .list-group-item{border-right:0;border-left:0;border-radius:0}.list-group-flush:first-child .list-group-item:first-child{border-top:0}.list-group-flush:last-child .list-group-item:last-child{border-bottom:0}.list-group-item-primary{color:#185578;background-color:#c5e6f8}.list-group-item-primary.list-group-item-action:hover,.list-group-item-primary.list-group-item-action:focus{color:#185578;background-color:#aedcf5}.list-group-item-primary.list-group-item-action.active{color:#fff;background-color:#185578;border-color:#185578}.list-group-item-secondary{color:#797b7c;background-color:#f9fafb}.list-group-item-secondary.list-group-item-action:hover,.list-group-item-secondary.list-group-item-action:focus{color:#797b7c;background-color:#eaedf1}.list-group-item-secondary.list-group-item-action.active{color:#fff;background-color:#797b7c;border-color:#797b7c}.list-group-item-success{color:#3c571e;background-color:#d8e7c8}.list-group-item-success.list-group-item-action:hover,.list-group-item-success.list-group-item-action:focus{color:#3c571e;background-color:#cbdfb6}.list-group-item-success.list-group-item-action.active{color:#fff;background-color:#3c571e;border-color:#3c571e}.list-group-item-info{color:#021f3c;background-color:#b8c8d8}.list-group-item-info.list-group-item-action:hover,.list-group-item-info.list-group-item-action:focus{color:#021f3c;background-color:#a8bbcf}.list-group-item-info.list-group-item-action.active{color:#fff;background-color:#021f3c;border-color:#021f3c}.list-group-item-warning{color:#732d00;background-color:#f5d0b8}.list-group-item-warning.list-group-item-action:hover,.list-group-item-warning.list-group-item-action:focus{color:#732d00;background-color:#f2c1a2}.list-group-item-warning.list-group-item-action.active{color:#fff;background-color:#732d00;border-color:#732d00}.list-group-item-danger{color:#670f12;background-color:#efbfc1}.list-group-item-danger.list-group-item-action:hover,.list-group-item-danger.list-group-item-action:focus{color:#670f12;background-color:#eaabad}.list-group-item-danger.list-group-item-action.active{color:#fff;background-color:#670f12;border-color:#670f12}.list-group-item-light{color:#818182;background-color:#fdfdfe}.list-group-item-light.list-group-item-action:hover,.list-group-item-light.list-group-item-action:focus{color:#818182;background-color:#ececf6}.list-group-item-light.list-group-item-action.active{color:#fff;background-color:#818182;border-color:#818182}.list-group-item-dark{color:#1b1e21;background-color:#c6c8ca}.list-group-item-dark.list-group-item-action:hover,.list-group-item-dark.list-group-item-action:focus{color:#1b1e21;background-color:#b9bbbe}.list-group-item-dark.list-group-item-action.active{color:#fff;background-color:#1b1e21;border-color:#1b1e21}.close{float:right;font-size:1.5rem;font-weight:700;line-height:1;color:#000;text-shadow:0 1px 0 #fff;opacity:.5}.close:not(:disabled):not(.disabled){cursor:pointer}.close:not(:disabled):not(.disabled):hover,.close:not(:disabled):not(.disabled):focus{color:#000;text-decoration:none;opacity:.75}button.close{padding:0;background-color:transparent;border:0;-webkit-appearance:none}.modal-open{overflow:hidden}.modal-open .modal{overflow-x:hidden;overflow-y:auto}.modal{position:fixed;top:0;right:0;bottom:0;left:0;z-index:1050;display:none;overflow:hidden;outline:0}.modal-dialog{position:relative;width:auto;margin:0.5rem;pointer-events:none}.modal.fade .modal-dialog{-webkit-transition:-webkit-transform 0.3s ease-out;transition:-webkit-transform 0.3s ease-out;transition:transform 0.3s ease-out;transition:transform 0.3s ease-out, -webkit-transform 0.3s ease-out;-webkit-transform:translate(0, -25%);transform:translate(0, -25%)}@media screen and (prefers-reduced-motion: reduce){.modal.fade .modal-dialog{-webkit-transition:none;transition:none}}.modal.show .modal-dialog{-webkit-transform:translate(0, 0);transform:translate(0, 0)}.modal-dialog-centered{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-align:center;-ms-flex-align:center;align-items:center;min-height:calc(100% - (0.5rem * 2))}.modal-dialog-centered::before{display:block;height:calc(100vh - (0.5rem * 2));content:""}.modal-content{position:relative;display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column;width:100%;pointer-events:auto;background-color:#fff;background-clip:padding-box;border:1px solid rgba(0,0,0,0.2);border-radius:0.3rem;outline:0}.modal-backdrop{position:fixed;top:0;right:0;bottom:0;left:0;z-index:1040;background-color:#000}.modal-backdrop.fade{opacity:0}.modal-backdrop.show{opacity:0.5}.modal-header{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-align:start;-ms-flex-align:start;align-items:flex-start;-webkit-box-pack:justify;-ms-flex-pack:justify;justify-content:space-between;padding:1rem;border-bottom:1px solid #e9ecef;border-top-left-radius:0.3rem;border-top-right-radius:0.3rem}.modal-header .close{padding:1rem;margin:-1rem -1rem -1rem auto}.modal-title{margin-bottom:0;line-height:1.5}.modal-body{position:relative;-webkit-box-flex:1;-ms-flex:1 1 auto;flex:1 1 auto;padding:1rem}.modal-footer{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-align:center;-ms-flex-align:center;align-items:center;-webkit-box-pack:end;-ms-flex-pack:end;justify-content:flex-end;padding:1rem;border-top:1px solid #e9ecef}.modal-footer>:not(:first-child){margin-left:.25rem}.modal-footer>:not(:last-child){margin-right:.25rem}.modal-scrollbar-measure{position:absolute;top:-9999px;width:50px;height:50px;overflow:scroll}@media (min-width: 576px){.modal-dialog{max-width:500px;margin:1.75rem auto}.modal-dialog-centered{min-height:calc(100% - (1.75rem * 2))}.modal-dialog-centered::before{height:calc(100vh - (1.75rem * 2))}.modal-sm{max-width:300px}}@media (min-width: 992px){.modal-lg{max-width:800px}}.tooltip{position:absolute;z-index:1070;display:block;margin:0;font-family:-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji";font-style:normal;font-weight:400;line-height:1.5;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;letter-spacing:normal;word-break:normal;word-spacing:normal;white-space:normal;line-break:auto;font-size:0.875rem;word-wrap:break-word;opacity:0}.tooltip.show{opacity:0.9}.tooltip .arrow{position:absolute;display:block;width:0.8rem;height:0.4rem}.tooltip .arrow::before{position:absolute;content:"";border-color:transparent;border-style:solid}.bs-tooltip-top,.bs-tooltip-auto[x-placement^="top"]{padding:0.4rem 0}.bs-tooltip-top .arrow,.bs-tooltip-auto[x-placement^="top"] .arrow{bottom:0}.bs-tooltip-top .arrow::before,.bs-tooltip-auto[x-placement^="top"] .arrow::before{top:0;border-width:0.4rem 0.4rem 0;border-top-color:#000}.bs-tooltip-right,.bs-tooltip-auto[x-placement^="right"]{padding:0 0.4rem}.bs-tooltip-right .arrow,.bs-tooltip-auto[x-placement^="right"] .arrow{left:0;width:0.4rem;height:0.8rem}.bs-tooltip-right .arrow::before,.bs-tooltip-auto[x-placement^="right"] .arrow::before{right:0;border-width:0.4rem 0.4rem 0.4rem 0;border-right-color:#000}.bs-tooltip-bottom,.bs-tooltip-auto[x-placement^="bottom"]{padding:0.4rem 0}.bs-tooltip-bottom .arrow,.bs-tooltip-auto[x-placement^="bottom"] .arrow{top:0}.bs-tooltip-bottom .arrow::before,.bs-tooltip-auto[x-placement^="bottom"] .arrow::before{bottom:0;border-width:0 0.4rem 0.4rem;border-bottom-color:#000}.bs-tooltip-left,.bs-tooltip-auto[x-placement^="left"]{padding:0 0.4rem}.bs-tooltip-left .arrow,.bs-tooltip-auto[x-placement^="left"] .arrow{right:0;width:0.4rem;height:0.8rem}.bs-tooltip-left .arrow::before,.bs-tooltip-auto[x-placement^="left"] .arrow::before{left:0;border-width:0.4rem 0 0.4rem 0.4rem;border-left-color:#000}.tooltip-inner{max-width:200px;padding:0.25rem 0.5rem;color:#fff;text-align:center;background-color:#000;border-radius:0.25rem}.popover{position:absolute;top:0;left:0;z-index:1060;display:block;max-width:276px;font-family:-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji";font-style:normal;font-weight:400;line-height:1.5;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;letter-spacing:normal;word-break:normal;word-spacing:normal;white-space:normal;line-break:auto;font-size:0.875rem;word-wrap:break-word;background-color:#fff;background-clip:padding-box;border:1px solid rgba(0,0,0,0.2);border-radius:0.3rem}.popover .arrow{position:absolute;display:block;width:1rem;height:0.5rem;margin:0 0.3rem}.popover .arrow::before,.popover .arrow::after{position:absolute;display:block;content:"";border-color:transparent;border-style:solid}.bs-popover-top,.bs-popover-auto[x-placement^="top"]{margin-bottom:0.5rem}.bs-popover-top .arrow,.bs-popover-auto[x-placement^="top"] .arrow{bottom:calc((0.5rem + 1px) * -1)}.bs-popover-top .arrow::before,.bs-popover-auto[x-placement^="top"] .arrow::before,.bs-popover-top .arrow::after,.bs-popover-auto[x-placement^="top"] .arrow::after{border-width:0.5rem 0.5rem 0}.bs-popover-top .arrow::before,.bs-popover-auto[x-placement^="top"] .arrow::before{bottom:0;border-top-color:rgba(0,0,0,0.25)}.bs-popover-top .arrow::after,.bs-popover-auto[x-placement^="top"] .arrow::after{bottom:1px;border-top-color:#fff}.bs-popover-right,.bs-popover-auto[x-placement^="right"]{margin-left:0.5rem}.bs-popover-right .arrow,.bs-popover-auto[x-placement^="right"] .arrow{left:calc((0.5rem + 1px) * -1);width:0.5rem;height:1rem;margin:0.3rem 0}.bs-popover-right .arrow::before,.bs-popover-auto[x-placement^="right"] .arrow::before,.bs-popover-right .arrow::after,.bs-popover-auto[x-placement^="right"] .arrow::after{border-width:0.5rem 0.5rem 0.5rem 0}.bs-popover-right .arrow::before,.bs-popover-auto[x-placement^="right"] .arrow::before{left:0;border-right-color:rgba(0,0,0,0.25)}.bs-popover-right .arrow::after,.bs-popover-auto[x-placement^="right"] .arrow::after{left:1px;border-right-color:#fff}.bs-popover-bottom,.bs-popover-auto[x-placement^="bottom"]{margin-top:0.5rem}.bs-popover-bottom .arrow,.bs-popover-auto[x-placement^="bottom"] .arrow{top:calc((0.5rem + 1px) * -1)}.bs-popover-bottom .arrow::before,.bs-popover-auto[x-placement^="bottom"] .arrow::before,.bs-popover-bottom .arrow::after,.bs-popover-auto[x-placement^="bottom"] .arrow::after{border-width:0 0.5rem 0.5rem 0.5rem}.bs-popover-bottom .arrow::before,.bs-popover-auto[x-placement^="bottom"] .arrow::before{top:0;border-bottom-color:rgba(0,0,0,0.25)}.bs-popover-bottom .arrow::after,.bs-popover-auto[x-placement^="bottom"] .arrow::after{top:1px;border-bottom-color:#fff}.bs-popover-bottom .popover-header::before,.bs-popover-auto[x-placement^="bottom"] .popover-header::before{position:absolute;top:0;left:50%;display:block;width:1rem;margin-left:-0.5rem;content:"";border-bottom:1px solid #f7f7f7}.bs-popover-left,.bs-popover-auto[x-placement^="left"]{margin-right:0.5rem}.bs-popover-left .arrow,.bs-popover-auto[x-placement^="left"] .arrow{right:calc((0.5rem + 1px) * -1);width:0.5rem;height:1rem;margin:0.3rem 0}.bs-popover-left .arrow::before,.bs-popover-auto[x-placement^="left"] .arrow::before,.bs-popover-left .arrow::after,.bs-popover-auto[x-placement^="left"] .arrow::after{border-width:0.5rem 0 0.5rem 0.5rem}.bs-popover-left .arrow::before,.bs-popover-auto[x-placement^="left"] .arrow::before{right:0;border-left-color:rgba(0,0,0,0.25)}.bs-popover-left .arrow::after,.bs-popover-auto[x-placement^="left"] .arrow::after{right:1px;border-left-color:#fff}.popover-header{padding:0.5rem 0.75rem;margin-bottom:0;font-size:1rem;color:#2FA4E7;background-color:#f7f7f7;border-bottom:1px solid #ebebeb;border-top-left-radius:calc(0.3rem - 1px);border-top-right-radius:calc(0.3rem - 1px)}.popover-header:empty{display:none}.popover-body{padding:0.5rem 0.75rem;color:#495057}.carousel{position:relative}.carousel-inner{position:relative;width:100%;overflow:hidden}.carousel-item{position:relative;display:none;-webkit-box-align:center;-ms-flex-align:center;align-items:center;width:100%;-webkit-backface-visibility:hidden;backface-visibility:hidden;-webkit-perspective:1000px;perspective:1000px}.carousel-item.active,.carousel-item-next,.carousel-item-prev{display:block;-webkit-transition:-webkit-transform 0.6s ease;transition:-webkit-transform 0.6s ease;transition:transform 0.6s ease;transition:transform 0.6s ease, -webkit-transform 0.6s ease}@media screen and (prefers-reduced-motion: reduce){.carousel-item.active,.carousel-item-next,.carousel-item-prev{-webkit-transition:none;transition:none}}.carousel-item-next,.carousel-item-prev{position:absolute;top:0}.carousel-item-next.carousel-item-left,.carousel-item-prev.carousel-item-right{-webkit-transform:translateX(0);transform:translateX(0)}@supports (-webkit-transform-style: preserve-3d) or (transform-style: preserve-3d){.carousel-item-next.carousel-item-left,.carousel-item-prev.carousel-item-right{-webkit-transform:translate3d(0, 0, 0);transform:translate3d(0, 0, 0)}}.carousel-item-next,.active.carousel-item-right{-webkit-transform:translateX(100%);transform:translateX(100%)}@supports (-webkit-transform-style: preserve-3d) or (transform-style: preserve-3d){.carousel-item-next,.active.carousel-item-right{-webkit-transform:translate3d(100%, 0, 0);transform:translate3d(100%, 0, 0)}}.carousel-item-prev,.active.carousel-item-left{-webkit-transform:translateX(-100%);transform:translateX(-100%)}@supports (-webkit-transform-style: preserve-3d) or (transform-style: preserve-3d){.carousel-item-prev,.active.carousel-item-left{-webkit-transform:translate3d(-100%, 0, 0);transform:translate3d(-100%, 0, 0)}}.carousel-fade .carousel-item{opacity:0;-webkit-transition-duration:.6s;transition-duration:.6s;-webkit-transition-property:opacity;transition-property:opacity}.carousel-fade .carousel-item.active,.carousel-fade .carousel-item-next.carousel-item-left,.carousel-fade .carousel-item-prev.carousel-item-right{opacity:1}.carousel-fade .active.carousel-item-left,.carousel-fade .active.carousel-item-right{opacity:0}.carousel-fade .carousel-item-next,.carousel-fade .carousel-item-prev,.carousel-fade .carousel-item.active,.carousel-fade .active.carousel-item-left,.carousel-fade .active.carousel-item-prev{-webkit-transform:translateX(0);transform:translateX(0)}@supports (-webkit-transform-style: preserve-3d) or (transform-style: preserve-3d){.carousel-fade .carousel-item-next,.carousel-fade .carousel-item-prev,.carousel-fade .carousel-item.active,.carousel-fade .active.carousel-item-left,.carousel-fade .active.carousel-item-prev{-webkit-transform:translate3d(0, 0, 0);transform:translate3d(0, 0, 0)}}.carousel-control-prev,.carousel-control-next{position:absolute;top:0;bottom:0;display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-align:center;-ms-flex-align:center;align-items:center;-webkit-box-pack:center;-ms-flex-pack:center;justify-content:center;width:15%;color:#fff;text-align:center;opacity:0.5}.carousel-control-prev:hover,.carousel-control-prev:focus,.carousel-control-next:hover,.carousel-control-next:focus{color:#fff;text-decoration:none;outline:0;opacity:.9}.carousel-control-prev{left:0}.carousel-control-next{right:0}.carousel-control-prev-icon,.carousel-control-next-icon{display:inline-block;width:20px;height:20px;background:transparent no-repeat center center;background-size:100% 100%}.carousel-control-prev-icon{background-image:url("data:image/svg+xml;charset=utf8,%3Csvg xmlns='http://www.w3.org/2000/svg' fill='%23fff' viewBox='0 0 8 8'%3E%3Cpath d='M5.25 0l-4 4 4 4 1.5-1.5-2.5-2.5 2.5-2.5-1.5-1.5z'/%3E%3C/svg%3E")}.carousel-control-next-icon{background-image:url("data:image/svg+xml;charset=utf8,%3Csvg xmlns='http://www.w3.org/2000/svg' fill='%23fff' viewBox='0 0 8 8'%3E%3Cpath d='M2.75 0l-1.5 1.5 2.5 2.5-2.5 2.5 1.5 1.5 4-4-4-4z'/%3E%3C/svg%3E")}.carousel-indicators{position:absolute;right:0;bottom:10px;left:0;z-index:15;display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-pack:center;-ms-flex-pack:center;justify-content:center;padding-left:0;margin-right:15%;margin-left:15%;list-style:none}.carousel-indicators li{position:relative;-webkit-box-flex:0;-ms-flex:0 1 auto;flex:0 1 auto;width:30px;height:3px;margin-right:3px;margin-left:3px;text-indent:-999px;cursor:pointer;background-color:rgba(255,255,255,0.5)}.carousel-indicators li::before{position:absolute;top:-10px;left:0;display:inline-block;width:100%;height:10px;content:""}.carousel-indicators li::after{position:absolute;bottom:-10px;left:0;display:inline-block;width:100%;height:10px;content:""}.carousel-indicators .active{background-color:#fff}.carousel-caption{position:absolute;right:15%;bottom:20px;left:15%;z-index:10;padding-top:20px;padding-bottom:20px;color:#fff;text-align:center}.align-baseline{vertical-align:baseline !important}.align-top{vertical-align:top !important}.align-middle{vertical-align:middle !important}.align-bottom{vertical-align:bottom !important}.align-text-bottom{vertical-align:text-bottom !important}.align-text-top{vertical-align:text-top !important}.bg-primary{background-color:#2FA4E7 !important}a.bg-primary:hover,a.bg-primary:focus,button.bg-primary:hover,button.bg-primary:focus{background-color:#178acc !important}.bg-secondary{background-color:#e9ecef !important}a.bg-secondary:hover,a.bg-secondary:focus,button.bg-secondary:hover,button.bg-secondary:focus{background-color:#cbd3da !important}.bg-success{background-color:#73A839 !important}a.bg-success:hover,a.bg-success:focus,button.bg-success:hover,button.bg-success:focus{background-color:#59822c !important}.bg-info{background-color:#033C73 !important}a.bg-info:hover,a.bg-info:focus,button.bg-info:hover,button.bg-info:focus{background-color:#022241 !important}.bg-warning{background-color:#DD5600 !important}a.bg-warning:hover,a.bg-warning:focus,button.bg-warning:hover,button.bg-warning:focus{background-color:#aa4200 !important}.bg-danger{background-color:#C71C22 !important}a.bg-danger:hover,a.bg-danger:focus,button.bg-danger:hover,button.bg-danger:focus{background-color:#9a161a !important}.bg-light{background-color:#f8f9fa !important}a.bg-light:hover,a.bg-light:focus,button.bg-light:hover,button.bg-light:focus{background-color:#dae0e5 !important}.bg-dark{background-color:#343a40 !important}a.bg-dark:hover,a.bg-dark:focus,button.bg-dark:hover,button.bg-dark:focus{background-color:#1d2124 !important}.bg-white{background-color:#fff !important}.bg-transparent{background-color:transparent !important}.border{border:1px solid #dee2e6 !important}.border-top{border-top:1px solid #dee2e6 !important}.border-right{border-right:1px solid #dee2e6 !important}.border-bottom{border-bottom:1px solid #dee2e6 !important}.border-left{border-left:1px solid #dee2e6 !important}.border-0{border:0 !important}.border-top-0{border-top:0 !important}.border-right-0{border-right:0 !important}.border-bottom-0{border-bottom:0 !important}.border-left-0{border-left:0 !important}.border-primary{border-color:#2FA4E7 !important}.border-secondary{border-color:#e9ecef !important}.border-success{border-color:#73A839 !important}.border-info{border-color:#033C73 !important}.border-warning{border-color:#DD5600 !important}.border-danger{border-color:#C71C22 !important}.border-light{border-color:#f8f9fa !important}.border-dark{border-color:#343a40 !important}.border-white{border-color:#fff !important}.rounded{border-radius:0.25rem !important}.rounded-top{border-top-left-radius:0.25rem !important;border-top-right-radius:0.25rem !important}.rounded-right{border-top-right-radius:0.25rem !important;border-bottom-right-radius:0.25rem !important}.rounded-bottom{border-bottom-right-radius:0.25rem !important;border-bottom-left-radius:0.25rem !important}.rounded-left{border-top-left-radius:0.25rem !important;border-bottom-left-radius:0.25rem !important}.rounded-circle{border-radius:50% !important}.rounded-0{border-radius:0 !important}.clearfix::after{display:block;clear:both;content:""}.d-none{display:none !important}.d-inline{display:inline !important}.d-inline-block{display:inline-block !important}.d-block{display:block !important}.d-table{display:table !important}.d-table-row{display:table-row !important}.d-table-cell{display:table-cell !important}.d-flex{display:-webkit-box !important;display:-ms-flexbox !important;display:flex !important}.d-inline-flex{display:-webkit-inline-box !important;display:-ms-inline-flexbox !important;display:inline-flex !important}@media (min-width: 576px){.d-sm-none{display:none !important}.d-sm-inline{display:inline !important}.d-sm-inline-block{display:inline-block !important}.d-sm-block{display:block !important}.d-sm-table{display:table !important}.d-sm-table-row{display:table-row !important}.d-sm-table-cell{display:table-cell !important}.d-sm-flex{display:-webkit-box !important;display:-ms-flexbox !important;display:flex !important}.d-sm-inline-flex{display:-webkit-inline-box !important;display:-ms-inline-flexbox !important;display:inline-flex !important}}@media (min-width: 768px){.d-md-none{display:none !important}.d-md-inline{display:inline !important}.d-md-inline-block{display:inline-block !important}.d-md-block{display:block !important}.d-md-table{display:table !important}.d-md-table-row{display:table-row !important}.d-md-table-cell{display:table-cell !important}.d-md-flex{display:-webkit-box !important;display:-ms-flexbox !important;display:flex !important}.d-md-inline-flex{display:-webkit-inline-box !important;display:-ms-inline-flexbox !important;display:inline-flex !important}}@media (min-width: 992px){.d-lg-none{display:none !important}.d-lg-inline{display:inline !important}.d-lg-inline-block{display:inline-block !important}.d-lg-block{display:block !important}.d-lg-table{display:table !important}.d-lg-table-row{display:table-row !important}.d-lg-table-cell{display:table-cell !important}.d-lg-flex{display:-webkit-box !important;display:-ms-flexbox !important;display:flex !important}.d-lg-inline-flex{display:-webkit-inline-box !important;display:-ms-inline-flexbox !important;display:inline-flex !important}}@media (min-width: 1200px){.d-xl-none{display:none !important}.d-xl-inline{display:inline !important}.d-xl-inline-block{display:inline-block !important}.d-xl-block{display:block !important}.d-xl-table{display:table !important}.d-xl-table-row{display:table-row !important}.d-xl-table-cell{display:table-cell !important}.d-xl-flex{display:-webkit-box !important;display:-ms-flexbox !important;display:flex !important}.d-xl-inline-flex{display:-webkit-inline-box !important;display:-ms-inline-flexbox !important;display:inline-flex !important}}@media print{.d-print-none{display:none !important}.d-print-inline{display:inline !important}.d-print-inline-block{display:inline-block !important}.d-print-block{display:block !important}.d-print-table{display:table !important}.d-print-table-row{display:table-row !important}.d-print-table-cell{display:table-cell !important}.d-print-flex{display:-webkit-box !important;display:-ms-flexbox !important;display:flex !important}.d-print-inline-flex{display:-webkit-inline-box !important;display:-ms-inline-flexbox !important;display:inline-flex !important}}.embed-responsive{position:relative;display:block;width:100%;padding:0;overflow:hidden}.embed-responsive::before{display:block;content:""}.embed-responsive .embed-responsive-item,.embed-responsive iframe,.embed-responsive embed,.embed-responsive object,.embed-responsive video{position:absolute;top:0;bottom:0;left:0;width:100%;height:100%;border:0}.embed-responsive-21by9::before{padding-top:42.8571428571%}.embed-responsive-16by9::before{padding-top:56.25%}.embed-responsive-4by3::before{padding-top:75%}.embed-responsive-1by1::before{padding-top:100%}.flex-row{-webkit-box-orient:horizontal !important;-webkit-box-direction:normal !important;-ms-flex-direction:row !important;flex-direction:row !important}.flex-column{-webkit-box-orient:vertical !important;-webkit-box-direction:normal !important;-ms-flex-direction:column !important;flex-direction:column !important}.flex-row-reverse{-webkit-box-orient:horizontal !important;-webkit-box-direction:reverse !important;-ms-flex-direction:row-reverse !important;flex-direction:row-reverse !important}.flex-column-reverse{-webkit-box-orient:vertical !important;-webkit-box-direction:reverse !important;-ms-flex-direction:column-reverse !important;flex-direction:column-reverse !important}.flex-wrap{-ms-flex-wrap:wrap !important;flex-wrap:wrap !important}.flex-nowrap{-ms-flex-wrap:nowrap !important;flex-wrap:nowrap !important}.flex-wrap-reverse{-ms-flex-wrap:wrap-reverse !important;flex-wrap:wrap-reverse !important}.flex-fill{-webkit-box-flex:1 !important;-ms-flex:1 1 auto !important;flex:1 1 auto !important}.flex-grow-0{-webkit-box-flex:0 !important;-ms-flex-positive:0 !important;flex-grow:0 !important}.flex-grow-1{-webkit-box-flex:1 !important;-ms-flex-positive:1 !important;flex-grow:1 !important}.flex-shrink-0{-ms-flex-negative:0 !important;flex-shrink:0 !important}.flex-shrink-1{-ms-flex-negative:1 !important;flex-shrink:1 !important}.justify-content-start{-webkit-box-pack:start !important;-ms-flex-pack:start !important;justify-content:flex-start !important}.justify-content-end{-webkit-box-pack:end !important;-ms-flex-pack:end !important;justify-content:flex-end !important}.justify-content-center{-webkit-box-pack:center !important;-ms-flex-pack:center !important;justify-content:center !important}.justify-content-between{-webkit-box-pack:justify !important;-ms-flex-pack:justify !important;justify-content:space-between !important}.justify-content-around{-ms-flex-pack:distribute !important;justify-content:space-around !important}.align-items-start{-webkit-box-align:start !important;-ms-flex-align:start !important;align-items:flex-start !important}.align-items-end{-webkit-box-align:end !important;-ms-flex-align:end !important;align-items:flex-end !important}.align-items-center{-webkit-box-align:center !important;-ms-flex-align:center !important;align-items:center !important}.align-items-baseline{-webkit-box-align:baseline !important;-ms-flex-align:baseline !important;align-items:baseline !important}.align-items-stretch{-webkit-box-align:stretch !important;-ms-flex-align:stretch !important;align-items:stretch !important}.align-content-start{-ms-flex-line-pack:start !important;align-content:flex-start !important}.align-content-end{-ms-flex-line-pack:end !important;align-content:flex-end !important}.align-content-center{-ms-flex-line-pack:center !important;align-content:center !important}.align-content-between{-ms-flex-line-pack:justify !important;align-content:space-between !important}.align-content-around{-ms-flex-line-pack:distribute !important;align-content:space-around !important}.align-content-stretch{-ms-flex-line-pack:stretch !important;align-content:stretch !important}.align-self-auto{-ms-flex-item-align:auto !important;align-self:auto !important}.align-self-start{-ms-flex-item-align:start !important;align-self:flex-start !important}.align-self-end{-ms-flex-item-align:end !important;align-self:flex-end !important}.align-self-center{-ms-flex-item-align:center !important;align-self:center !important}.align-self-baseline{-ms-flex-item-align:baseline !important;align-self:baseline !important}.align-self-stretch{-ms-flex-item-align:stretch !important;align-self:stretch !important}@media (min-width: 576px){.flex-sm-row{-webkit-box-orient:horizontal !important;-webkit-box-direction:normal !important;-ms-flex-direction:row !important;flex-direction:row !important}.flex-sm-column{-webkit-box-orient:vertical !important;-webkit-box-direction:normal !important;-ms-flex-direction:column !important;flex-direction:column !important}.flex-sm-row-reverse{-webkit-box-orient:horizontal !important;-webkit-box-direction:reverse !important;-ms-flex-direction:row-reverse !important;flex-direction:row-reverse !important}.flex-sm-column-reverse{-webkit-box-orient:vertical !important;-webkit-box-direction:reverse !important;-ms-flex-direction:column-reverse !important;flex-direction:column-reverse !important}.flex-sm-wrap{-ms-flex-wrap:wrap !important;flex-wrap:wrap !important}.flex-sm-nowrap{-ms-flex-wrap:nowrap !important;flex-wrap:nowrap !important}.flex-sm-wrap-reverse{-ms-flex-wrap:wrap-reverse !important;flex-wrap:wrap-reverse !important}.flex-sm-fill{-webkit-box-flex:1 !important;-ms-flex:1 1 auto !important;flex:1 1 auto !important}.flex-sm-grow-0{-webkit-box-flex:0 !important;-ms-flex-positive:0 !important;flex-grow:0 !important}.flex-sm-grow-1{-webkit-box-flex:1 !important;-ms-flex-positive:1 !important;flex-grow:1 !important}.flex-sm-shrink-0{-ms-flex-negative:0 !important;flex-shrink:0 !important}.flex-sm-shrink-1{-ms-flex-negative:1 !important;flex-shrink:1 !important}.justify-content-sm-start{-webkit-box-pack:start !important;-ms-flex-pack:start !important;justify-content:flex-start !important}.justify-content-sm-end{-webkit-box-pack:end !important;-ms-flex-pack:end !important;justify-content:flex-end !important}.justify-content-sm-center{-webkit-box-pack:center !important;-ms-flex-pack:center !important;justify-content:center !important}.justify-content-sm-between{-webkit-box-pack:justify !important;-ms-flex-pack:justify !important;justify-content:space-between !important}.justify-content-sm-around{-ms-flex-pack:distribute !important;justify-content:space-around !important}.align-items-sm-start{-webkit-box-align:start !important;-ms-flex-align:start !important;align-items:flex-start !important}.align-items-sm-end{-webkit-box-align:end !important;-ms-flex-align:end !important;align-items:flex-end !important}.align-items-sm-center{-webkit-box-align:center !important;-ms-flex-align:center !important;align-items:center !important}.align-items-sm-baseline{-webkit-box-align:baseline !important;-ms-flex-align:baseline !important;align-items:baseline !important}.align-items-sm-stretch{-webkit-box-align:stretch !important;-ms-flex-align:stretch !important;align-items:stretch !important}.align-content-sm-start{-ms-flex-line-pack:start !important;align-content:flex-start !important}.align-content-sm-end{-ms-flex-line-pack:end !important;align-content:flex-end !important}.align-content-sm-center{-ms-flex-line-pack:center !important;align-content:center !important}.align-content-sm-between{-ms-flex-line-pack:justify !important;align-content:space-between !important}.align-content-sm-around{-ms-flex-line-pack:distribute !important;align-content:space-around !important}.align-content-sm-stretch{-ms-flex-line-pack:stretch !important;align-content:stretch !important}.align-self-sm-auto{-ms-flex-item-align:auto !important;align-self:auto !important}.align-self-sm-start{-ms-flex-item-align:start !important;align-self:flex-start !important}.align-self-sm-end{-ms-flex-item-align:end !important;align-self:flex-end !important}.align-self-sm-center{-ms-flex-item-align:center !important;align-self:center !important}.align-self-sm-baseline{-ms-flex-item-align:baseline !important;align-self:baseline !important}.align-self-sm-stretch{-ms-flex-item-align:stretch !important;align-self:stretch !important}}@media (min-width: 768px){.flex-md-row{-webkit-box-orient:horizontal !important;-webkit-box-direction:normal !important;-ms-flex-direction:row !important;flex-direction:row !important}.flex-md-column{-webkit-box-orient:vertical !important;-webkit-box-direction:normal !important;-ms-flex-direction:column !important;flex-direction:column !important}.flex-md-row-reverse{-webkit-box-orient:horizontal !important;-webkit-box-direction:reverse !important;-ms-flex-direction:row-reverse !important;flex-direction:row-reverse !important}.flex-md-column-reverse{-webkit-box-orient:vertical !important;-webkit-box-direction:reverse !important;-ms-flex-direction:column-reverse !important;flex-direction:column-reverse !important}.flex-md-wrap{-ms-flex-wrap:wrap !important;flex-wrap:wrap !important}.flex-md-nowrap{-ms-flex-wrap:nowrap !important;flex-wrap:nowrap !important}.flex-md-wrap-reverse{-ms-flex-wrap:wrap-reverse !important;flex-wrap:wrap-reverse !important}.flex-md-fill{-webkit-box-flex:1 !important;-ms-flex:1 1 auto !important;flex:1 1 auto !important}.flex-md-grow-0{-webkit-box-flex:0 !important;-ms-flex-positive:0 !important;flex-grow:0 !important}.flex-md-grow-1{-webkit-box-flex:1 !important;-ms-flex-positive:1 !important;flex-grow:1 !important}.flex-md-shrink-0{-ms-flex-negative:0 !important;flex-shrink:0 !important}.flex-md-shrink-1{-ms-flex-negative:1 !important;flex-shrink:1 !important}.justify-content-md-start{-webkit-box-pack:start !important;-ms-flex-pack:start !important;justify-content:flex-start !important}.justify-content-md-end{-webkit-box-pack:end !important;-ms-flex-pack:end !important;justify-content:flex-end !important}.justify-content-md-center{-webkit-box-pack:center !important;-ms-flex-pack:center !important;justify-content:center !important}.justify-content-md-between{-webkit-box-pack:justify !important;-ms-flex-pack:justify !important;justify-content:space-between !important}.justify-content-md-around{-ms-flex-pack:distribute !important;justify-content:space-around !important}.align-items-md-start{-webkit-box-align:start !important;-ms-flex-align:start !important;align-items:flex-start !important}.align-items-md-end{-webkit-box-align:end !important;-ms-flex-align:end !important;align-items:flex-end !important}.align-items-md-center{-webkit-box-align:center !important;-ms-flex-align:center !important;align-items:center !important}.align-items-md-baseline{-webkit-box-align:baseline !important;-ms-flex-align:baseline !important;align-items:baseline !important}.align-items-md-stretch{-webkit-box-align:stretch !important;-ms-flex-align:stretch !important;align-items:stretch !important}.align-content-md-start{-ms-flex-line-pack:start !important;align-content:flex-start !important}.align-content-md-end{-ms-flex-line-pack:end !important;align-content:flex-end !important}.align-content-md-center{-ms-flex-line-pack:center !important;align-content:center !important}.align-content-md-between{-ms-flex-line-pack:justify !important;align-content:space-between !important}.align-content-md-around{-ms-flex-line-pack:distribute !important;align-content:space-around !important}.align-content-md-stretch{-ms-flex-line-pack:stretch !important;align-content:stretch !important}.align-self-md-auto{-ms-flex-item-align:auto !important;align-self:auto !important}.align-self-md-start{-ms-flex-item-align:start !important;align-self:flex-start !important}.align-self-md-end{-ms-flex-item-align:end !important;align-self:flex-end !important}.align-self-md-center{-ms-flex-item-align:center !important;align-self:center !important}.align-self-md-baseline{-ms-flex-item-align:baseline !important;align-self:baseline !important}.align-self-md-stretch{-ms-flex-item-align:stretch !important;align-self:stretch !important}}@media (min-width: 992px){.flex-lg-row{-webkit-box-orient:horizontal !important;-webkit-box-direction:normal !important;-ms-flex-direction:row !important;flex-direction:row !important}.flex-lg-column{-webkit-box-orient:vertical !important;-webkit-box-direction:normal !important;-ms-flex-direction:column !important;flex-direction:column !important}.flex-lg-row-reverse{-webkit-box-orient:horizontal !important;-webkit-box-direction:reverse !important;-ms-flex-direction:row-reverse !important;flex-direction:row-reverse !important}.flex-lg-column-reverse{-webkit-box-orient:vertical !important;-webkit-box-direction:reverse !important;-ms-flex-direction:column-reverse !important;flex-direction:column-reverse !important}.flex-lg-wrap{-ms-flex-wrap:wrap !important;flex-wrap:wrap !important}.flex-lg-nowrap{-ms-flex-wrap:nowrap !important;flex-wrap:nowrap !important}.flex-lg-wrap-reverse{-ms-flex-wrap:wrap-reverse !important;flex-wrap:wrap-reverse !important}.flex-lg-fill{-webkit-box-flex:1 !important;-ms-flex:1 1 auto !important;flex:1 1 auto !important}.flex-lg-grow-0{-webkit-box-flex:0 !important;-ms-flex-positive:0 !important;flex-grow:0 !important}.flex-lg-grow-1{-webkit-box-flex:1 !important;-ms-flex-positive:1 !important;flex-grow:1 !important}.flex-lg-shrink-0{-ms-flex-negative:0 !important;flex-shrink:0 !important}.flex-lg-shrink-1{-ms-flex-negative:1 !important;flex-shrink:1 !important}.justify-content-lg-start{-webkit-box-pack:start !important;-ms-flex-pack:start !important;justify-content:flex-start !important}.justify-content-lg-end{-webkit-box-pack:end !important;-ms-flex-pack:end !important;justify-content:flex-end !important}.justify-content-lg-center{-webkit-box-pack:center !important;-ms-flex-pack:center !important;justify-content:center !important}.justify-content-lg-between{-webkit-box-pack:justify !important;-ms-flex-pack:justify !important;justify-content:space-between !important}.justify-content-lg-around{-ms-flex-pack:distribute !important;justify-content:space-around !important}.align-items-lg-start{-webkit-box-align:start !important;-ms-flex-align:start !important;align-items:flex-start !important}.align-items-lg-end{-webkit-box-align:end !important;-ms-flex-align:end !important;align-items:flex-end !important}.align-items-lg-center{-webkit-box-align:center !important;-ms-flex-align:center !important;align-items:center !important}.align-items-lg-baseline{-webkit-box-align:baseline !important;-ms-flex-align:baseline !important;align-items:baseline !important}.align-items-lg-stretch{-webkit-box-align:stretch !important;-ms-flex-align:stretch !important;align-items:stretch !important}.align-content-lg-start{-ms-flex-line-pack:start !important;align-content:flex-start !important}.align-content-lg-end{-ms-flex-line-pack:end !important;align-content:flex-end !important}.align-content-lg-center{-ms-flex-line-pack:center !important;align-content:center !important}.align-content-lg-between{-ms-flex-line-pack:justify !important;align-content:space-between !important}.align-content-lg-around{-ms-flex-line-pack:distribute !important;align-content:space-around !important}.align-content-lg-stretch{-ms-flex-line-pack:stretch !important;align-content:stretch !important}.align-self-lg-auto{-ms-flex-item-align:auto !important;align-self:auto !important}.align-self-lg-start{-ms-flex-item-align:start !important;align-self:flex-start !important}.align-self-lg-end{-ms-flex-item-align:end !important;align-self:flex-end !important}.align-self-lg-center{-ms-flex-item-align:center !important;align-self:center !important}.align-self-lg-baseline{-ms-flex-item-align:baseline !important;align-self:baseline !important}.align-self-lg-stretch{-ms-flex-item-align:stretch !important;align-self:stretch !important}}@media (min-width: 1200px){.flex-xl-row{-webkit-box-orient:horizontal !important;-webkit-box-direction:normal !important;-ms-flex-direction:row !important;flex-direction:row !important}.flex-xl-column{-webkit-box-orient:vertical !important;-webkit-box-direction:normal !important;-ms-flex-direction:column !important;flex-direction:column !important}.flex-xl-row-reverse{-webkit-box-orient:horizontal !important;-webkit-box-direction:reverse !important;-ms-flex-direction:row-reverse !important;flex-direction:row-reverse !important}.flex-xl-column-reverse{-webkit-box-orient:vertical !important;-webkit-box-direction:reverse !important;-ms-flex-direction:column-reverse !important;flex-direction:column-reverse !important}.flex-xl-wrap{-ms-flex-wrap:wrap !important;flex-wrap:wrap !important}.flex-xl-nowrap{-ms-flex-wrap:nowrap !important;flex-wrap:nowrap !important}.flex-xl-wrap-reverse{-ms-flex-wrap:wrap-reverse !important;flex-wrap:wrap-reverse !important}.flex-xl-fill{-webkit-box-flex:1 !important;-ms-flex:1 1 auto !important;flex:1 1 auto !important}.flex-xl-grow-0{-webkit-box-flex:0 !important;-ms-flex-positive:0 !important;flex-grow:0 !important}.flex-xl-grow-1{-webkit-box-flex:1 !important;-ms-flex-positive:1 !important;flex-grow:1 !important}.flex-xl-shrink-0{-ms-flex-negative:0 !important;flex-shrink:0 !important}.flex-xl-shrink-1{-ms-flex-negative:1 !important;flex-shrink:1 !important}.justify-content-xl-start{-webkit-box-pack:start !important;-ms-flex-pack:start !important;justify-content:flex-start !important}.justify-content-xl-end{-webkit-box-pack:end !important;-ms-flex-pack:end !important;justify-content:flex-end !important}.justify-content-xl-center{-webkit-box-pack:center !important;-ms-flex-pack:center !important;justify-content:center !important}.justify-content-xl-between{-webkit-box-pack:justify !important;-ms-flex-pack:justify !important;justify-content:space-between !important}.justify-content-xl-around{-ms-flex-pack:distribute !important;justify-content:space-around !important}.align-items-xl-start{-webkit-box-align:start !important;-ms-flex-align:start !important;align-items:flex-start !important}.align-items-xl-end{-webkit-box-align:end !important;-ms-flex-align:end !important;align-items:flex-end !important}.align-items-xl-center{-webkit-box-align:center !important;-ms-flex-align:center !important;align-items:center !important}.align-items-xl-baseline{-webkit-box-align:baseline !important;-ms-flex-align:baseline !important;align-items:baseline !important}.align-items-xl-stretch{-webkit-box-align:stretch !important;-ms-flex-align:stretch !important;align-items:stretch !important}.align-content-xl-start{-ms-flex-line-pack:start !important;align-content:flex-start !important}.align-content-xl-end{-ms-flex-line-pack:end !important;align-content:flex-end !important}.align-content-xl-center{-ms-flex-line-pack:center !important;align-content:center !important}.align-content-xl-between{-ms-flex-line-pack:justify !important;align-content:space-between !important}.align-content-xl-around{-ms-flex-line-pack:distribute !important;align-content:space-around !important}.align-content-xl-stretch{-ms-flex-line-pack:stretch !important;align-content:stretch !important}.align-self-xl-auto{-ms-flex-item-align:auto !important;align-self:auto !important}.align-self-xl-start{-ms-flex-item-align:start !important;align-self:flex-start !important}.align-self-xl-end{-ms-flex-item-align:end !important;align-self:flex-end !important}.align-self-xl-center{-ms-flex-item-align:center !important;align-self:center !important}.align-self-xl-baseline{-ms-flex-item-align:baseline !important;align-self:baseline !important}.align-self-xl-stretch{-ms-flex-item-align:stretch !important;align-self:stretch !important}}.float-left{float:left !important}.float-right{float:right !important}.float-none{float:none !important}@media (min-width: 576px){.float-sm-left{float:left !important}.float-sm-right{float:right !important}.float-sm-none{float:none !important}}@media (min-width: 768px){.float-md-left{float:left !important}.float-md-right{float:right !important}.float-md-none{float:none !important}}@media (min-width: 992px){.float-lg-left{float:left !important}.float-lg-right{float:right !important}.float-lg-none{float:none !important}}@media (min-width: 1200px){.float-xl-left{float:left !important}.float-xl-right{float:right !important}.float-xl-none{float:none !important}}.position-static{position:static !important}.position-relative{position:relative !important}.position-absolute{position:absolute !important}.position-fixed{position:fixed !important}.position-sticky{position:-webkit-sticky !important;position:sticky !important}.fixed-top{position:fixed;top:0;right:0;left:0;z-index:1030}.fixed-bottom{position:fixed;right:0;bottom:0;left:0;z-index:1030}@supports (position: -webkit-sticky) or (position: sticky){.sticky-top{position:-webkit-sticky;position:sticky;top:0;z-index:1020}}.sr-only{position:absolute;width:1px;height:1px;padding:0;overflow:hidden;clip:rect(0, 0, 0, 0);white-space:nowrap;border:0}.sr-only-focusable:active,.sr-only-focusable:focus{position:static;width:auto;height:auto;overflow:visible;clip:auto;white-space:normal}.shadow-sm{-webkit-box-shadow:0 0.125rem 0.25rem rgba(0,0,0,0.075) !important;box-shadow:0 0.125rem 0.25rem rgba(0,0,0,0.075) !important}.shadow{-webkit-box-shadow:0 0.5rem 1rem rgba(0,0,0,0.15) !important;box-shadow:0 0.5rem 1rem rgba(0,0,0,0.15) !important}.shadow-lg{-webkit-box-shadow:0 1rem 3rem rgba(0,0,0,0.175) !important;box-shadow:0 1rem 3rem rgba(0,0,0,0.175) !important}.shadow-none{-webkit-box-shadow:none !important;box-shadow:none !important}.w-25{width:25% !important}.w-50{width:50% !important}.w-75{width:75% !important}.w-100{width:100% !important}.w-auto{width:auto !important}.h-25{height:25% !important}.h-50{height:50% !important}.h-75{height:75% !important}.h-100{height:100% !important}.h-auto{height:auto !important}.mw-100{max-width:100% !important}.mh-100{max-height:100% !important}.m-0{margin:0 !important}.mt-0,.my-0{margin-top:0 !important}.mr-0,.mx-0{margin-right:0 !important}.mb-0,.my-0{margin-bottom:0 !important}.ml-0,.mx-0{margin-left:0 !important}.m-1{margin:0.25rem !important}.mt-1,.my-1{margin-top:0.25rem !important}.mr-1,.mx-1{margin-right:0.25rem !important}.mb-1,.my-1{margin-bottom:0.25rem !important}.ml-1,.mx-1{margin-left:0.25rem !important}.m-2{margin:0.5rem !important}.mt-2,.my-2{margin-top:0.5rem !important}.mr-2,.mx-2{margin-right:0.5rem !important}.mb-2,.my-2{margin-bottom:0.5rem !important}.ml-2,.mx-2{margin-left:0.5rem !important}.m-3{margin:1rem !important}.mt-3,.my-3{margin-top:1rem !important}.mr-3,.mx-3{margin-right:1rem !important}.mb-3,.my-3{margin-bottom:1rem !important}.ml-3,.mx-3{margin-left:1rem !important}.m-4{margin:1.5rem !important}.mt-4,.my-4{margin-top:1.5rem !important}.mr-4,.mx-4{margin-right:1.5rem !important}.mb-4,.my-4{margin-bottom:1.5rem !important}.ml-4,.mx-4{margin-left:1.5rem !important}.m-5{margin:3rem !important}.mt-5,.my-5{margin-top:3rem !important}.mr-5,.mx-5{margin-right:3rem !important}.mb-5,.my-5{margin-bottom:3rem !important}.ml-5,.mx-5{margin-left:3rem !important}.p-0{padding:0 !important}.pt-0,.py-0{padding-top:0 !important}.pr-0,.px-0{padding-right:0 !important}.pb-0,.py-0{padding-bottom:0 !important}.pl-0,.px-0{padding-left:0 !important}.p-1{padding:0.25rem !important}.pt-1,.py-1{padding-top:0.25rem !important}.pr-1,.px-1{padding-right:0.25rem !important}.pb-1,.py-1{padding-bottom:0.25rem !important}.pl-1,.px-1{padding-left:0.25rem !important}.p-2{padding:0.5rem !important}.pt-2,.py-2{padding-top:0.5rem !important}.pr-2,.px-2{padding-right:0.5rem !important}.pb-2,.py-2{padding-bottom:0.5rem !important}.pl-2,.px-2{padding-left:0.5rem !important}.p-3{padding:1rem !important}.pt-3,.py-3{padding-top:1rem !important}.pr-3,.px-3{padding-right:1rem !important}.pb-3,.py-3{padding-bottom:1rem !important}.pl-3,.px-3{padding-left:1rem !important}.p-4{padding:1.5rem !important}.pt-4,.py-4{padding-top:1.5rem !important}.pr-4,.px-4{padding-right:1.5rem !important}.pb-4,.py-4{padding-bottom:1.5rem !important}.pl-4,.px-4{padding-left:1.5rem !important}.p-5{padding:3rem !important}.pt-5,.py-5{padding-top:3rem !important}.pr-5,.px-5{padding-right:3rem !important}.pb-5,.py-5{padding-bottom:3rem !important}.pl-5,.px-5{padding-left:3rem !important}.m-auto{margin:auto !important}.mt-auto,.my-auto{margin-top:auto !important}.mr-auto,.mx-auto{margin-right:auto !important}.mb-auto,.my-auto{margin-bottom:auto !important}.ml-auto,.mx-auto{margin-left:auto !important}@media (min-width: 576px){.m-sm-0{margin:0 !important}.mt-sm-0,.my-sm-0{margin-top:0 !important}.mr-sm-0,.mx-sm-0{margin-right:0 !important}.mb-sm-0,.my-sm-0{margin-bottom:0 !important}.ml-sm-0,.mx-sm-0{margin-left:0 !important}.m-sm-1{margin:0.25rem !important}.mt-sm-1,.my-sm-1{margin-top:0.25rem !important}.mr-sm-1,.mx-sm-1{margin-right:0.25rem !important}.mb-sm-1,.my-sm-1{margin-bottom:0.25rem !important}.ml-sm-1,.mx-sm-1{margin-left:0.25rem !important}.m-sm-2{margin:0.5rem !important}.mt-sm-2,.my-sm-2{margin-top:0.5rem !important}.mr-sm-2,.mx-sm-2{margin-right:0.5rem !important}.mb-sm-2,.my-sm-2{margin-bottom:0.5rem !important}.ml-sm-2,.mx-sm-2{margin-left:0.5rem !important}.m-sm-3{margin:1rem !important}.mt-sm-3,.my-sm-3{margin-top:1rem !important}.mr-sm-3,.mx-sm-3{margin-right:1rem !important}.mb-sm-3,.my-sm-3{margin-bottom:1rem !important}.ml-sm-3,.mx-sm-3{margin-left:1rem !important}.m-sm-4{margin:1.5rem !important}.mt-sm-4,.my-sm-4{margin-top:1.5rem !important}.mr-sm-4,.mx-sm-4{margin-right:1.5rem !important}.mb-sm-4,.my-sm-4{margin-bottom:1.5rem !important}.ml-sm-4,.mx-sm-4{margin-left:1.5rem !important}.m-sm-5{margin:3rem !important}.mt-sm-5,.my-sm-5{margin-top:3rem !important}.mr-sm-5,.mx-sm-5{margin-right:3rem !important}.mb-sm-5,.my-sm-5{margin-bottom:3rem !important}.ml-sm-5,.mx-sm-5{margin-left:3rem !important}.p-sm-0{padding:0 !important}.pt-sm-0,.py-sm-0{padding-top:0 !important}.pr-sm-0,.px-sm-0{padding-right:0 !important}.pb-sm-0,.py-sm-0{padding-bottom:0 !important}.pl-sm-0,.px-sm-0{padding-left:0 !important}.p-sm-1{padding:0.25rem !important}.pt-sm-1,.py-sm-1{padding-top:0.25rem !important}.pr-sm-1,.px-sm-1{padding-right:0.25rem !important}.pb-sm-1,.py-sm-1{padding-bottom:0.25rem !important}.pl-sm-1,.px-sm-1{padding-left:0.25rem !important}.p-sm-2{padding:0.5rem !important}.pt-sm-2,.py-sm-2{padding-top:0.5rem !important}.pr-sm-2,.px-sm-2{padding-right:0.5rem !important}.pb-sm-2,.py-sm-2{padding-bottom:0.5rem !important}.pl-sm-2,.px-sm-2{padding-left:0.5rem !important}.p-sm-3{padding:1rem !important}.pt-sm-3,.py-sm-3{padding-top:1rem !important}.pr-sm-3,.px-sm-3{padding-right:1rem !important}.pb-sm-3,.py-sm-3{padding-bottom:1rem !important}.pl-sm-3,.px-sm-3{padding-left:1rem !important}.p-sm-4{padding:1.5rem !important}.pt-sm-4,.py-sm-4{padding-top:1.5rem !important}.pr-sm-4,.px-sm-4{padding-right:1.5rem !important}.pb-sm-4,.py-sm-4{padding-bottom:1.5rem !important}.pl-sm-4,.px-sm-4{padding-left:1.5rem !important}.p-sm-5{padding:3rem !important}.pt-sm-5,.py-sm-5{padding-top:3rem !important}.pr-sm-5,.px-sm-5{padding-right:3rem !important}.pb-sm-5,.py-sm-5{padding-bottom:3rem !important}.pl-sm-5,.px-sm-5{padding-left:3rem !important}.m-sm-auto{margin:auto !important}.mt-sm-auto,.my-sm-auto{margin-top:auto !important}.mr-sm-auto,.mx-sm-auto{margin-right:auto !important}.mb-sm-auto,.my-sm-auto{margin-bottom:auto !important}.ml-sm-auto,.mx-sm-auto{margin-left:auto !important}}@media (min-width: 768px){.m-md-0{margin:0 !important}.mt-md-0,.my-md-0{margin-top:0 !important}.mr-md-0,.mx-md-0{margin-right:0 !important}.mb-md-0,.my-md-0{margin-bottom:0 !important}.ml-md-0,.mx-md-0{margin-left:0 !important}.m-md-1{margin:0.25rem !important}.mt-md-1,.my-md-1{margin-top:0.25rem !important}.mr-md-1,.mx-md-1{margin-right:0.25rem !important}.mb-md-1,.my-md-1{margin-bottom:0.25rem !important}.ml-md-1,.mx-md-1{margin-left:0.25rem !important}.m-md-2{margin:0.5rem !important}.mt-md-2,.my-md-2{margin-top:0.5rem !important}.mr-md-2,.mx-md-2{margin-right:0.5rem !important}.mb-md-2,.my-md-2{margin-bottom:0.5rem !important}.ml-md-2,.mx-md-2{margin-left:0.5rem !important}.m-md-3{margin:1rem !important}.mt-md-3,.my-md-3{margin-top:1rem !important}.mr-md-3,.mx-md-3{margin-right:1rem !important}.mb-md-3,.my-md-3{margin-bottom:1rem !important}.ml-md-3,.mx-md-3{margin-left:1rem !important}.m-md-4{margin:1.5rem !important}.mt-md-4,.my-md-4{margin-top:1.5rem !important}.mr-md-4,.mx-md-4{margin-right:1.5rem !important}.mb-md-4,.my-md-4{margin-bottom:1.5rem !important}.ml-md-4,.mx-md-4{margin-left:1.5rem !important}.m-md-5{margin:3rem !important}.mt-md-5,.my-md-5{margin-top:3rem !important}.mr-md-5,.mx-md-5{margin-right:3rem !important}.mb-md-5,.my-md-5{margin-bottom:3rem !important}.ml-md-5,.mx-md-5{margin-left:3rem !important}.p-md-0{padding:0 !important}.pt-md-0,.py-md-0{padding-top:0 !important}.pr-md-0,.px-md-0{padding-right:0 !important}.pb-md-0,.py-md-0{padding-bottom:0 !important}.pl-md-0,.px-md-0{padding-left:0 !important}.p-md-1{padding:0.25rem !important}.pt-md-1,.py-md-1{padding-top:0.25rem !important}.pr-md-1,.px-md-1{padding-right:0.25rem !important}.pb-md-1,.py-md-1{padding-bottom:0.25rem !important}.pl-md-1,.px-md-1{padding-left:0.25rem !important}.p-md-2{padding:0.5rem !important}.pt-md-2,.py-md-2{padding-top:0.5rem !important}.pr-md-2,.px-md-2{padding-right:0.5rem !important}.pb-md-2,.py-md-2{padding-bottom:0.5rem !important}.pl-md-2,.px-md-2{padding-left:0.5rem !important}.p-md-3{padding:1rem !important}.pt-md-3,.py-md-3{padding-top:1rem !important}.pr-md-3,.px-md-3{padding-right:1rem !important}.pb-md-3,.py-md-3{padding-bottom:1rem !important}.pl-md-3,.px-md-3{padding-left:1rem !important}.p-md-4{padding:1.5rem !important}.pt-md-4,.py-md-4{padding-top:1.5rem !important}.pr-md-4,.px-md-4{padding-right:1.5rem !important}.pb-md-4,.py-md-4{padding-bottom:1.5rem !important}.pl-md-4,.px-md-4{padding-left:1.5rem !important}.p-md-5{padding:3rem !important}.pt-md-5,.py-md-5{padding-top:3rem !important}.pr-md-5,.px-md-5{padding-right:3rem !important}.pb-md-5,.py-md-5{padding-bottom:3rem !important}.pl-md-5,.px-md-5{padding-left:3rem !important}.m-md-auto{margin:auto !important}.mt-md-auto,.my-md-auto{margin-top:auto !important}.mr-md-auto,.mx-md-auto{margin-right:auto !important}.mb-md-auto,.my-md-auto{margin-bottom:auto !important}.ml-md-auto,.mx-md-auto{margin-left:auto !important}}@media (min-width: 992px){.m-lg-0{margin:0 !important}.mt-lg-0,.my-lg-0{margin-top:0 !important}.mr-lg-0,.mx-lg-0{margin-right:0 !important}.mb-lg-0,.my-lg-0{margin-bottom:0 !important}.ml-lg-0,.mx-lg-0{margin-left:0 !important}.m-lg-1{margin:0.25rem !important}.mt-lg-1,.my-lg-1{margin-top:0.25rem !important}.mr-lg-1,.mx-lg-1{margin-right:0.25rem !important}.mb-lg-1,.my-lg-1{margin-bottom:0.25rem !important}.ml-lg-1,.mx-lg-1{margin-left:0.25rem !important}.m-lg-2{margin:0.5rem !important}.mt-lg-2,.my-lg-2{margin-top:0.5rem !important}.mr-lg-2,.mx-lg-2{margin-right:0.5rem !important}.mb-lg-2,.my-lg-2{margin-bottom:0.5rem !important}.ml-lg-2,.mx-lg-2{margin-left:0.5rem !important}.m-lg-3{margin:1rem !important}.mt-lg-3,.my-lg-3{margin-top:1rem !important}.mr-lg-3,.mx-lg-3{margin-right:1rem !important}.mb-lg-3,.my-lg-3{margin-bottom:1rem !important}.ml-lg-3,.mx-lg-3{margin-left:1rem !important}.m-lg-4{margin:1.5rem !important}.mt-lg-4,.my-lg-4{margin-top:1.5rem !important}.mr-lg-4,.mx-lg-4{margin-right:1.5rem !important}.mb-lg-4,.my-lg-4{margin-bottom:1.5rem !important}.ml-lg-4,.mx-lg-4{margin-left:1.5rem !important}.m-lg-5{margin:3rem !important}.mt-lg-5,.my-lg-5{margin-top:3rem !important}.mr-lg-5,.mx-lg-5{margin-right:3rem !important}.mb-lg-5,.my-lg-5{margin-bottom:3rem !important}.ml-lg-5,.mx-lg-5{margin-left:3rem !important}.p-lg-0{padding:0 !important}.pt-lg-0,.py-lg-0{padding-top:0 !important}.pr-lg-0,.px-lg-0{padding-right:0 !important}.pb-lg-0,.py-lg-0{padding-bottom:0 !important}.pl-lg-0,.px-lg-0{padding-left:0 !important}.p-lg-1{padding:0.25rem !important}.pt-lg-1,.py-lg-1{padding-top:0.25rem !important}.pr-lg-1,.px-lg-1{padding-right:0.25rem !important}.pb-lg-1,.py-lg-1{padding-bottom:0.25rem !important}.pl-lg-1,.px-lg-1{padding-left:0.25rem !important}.p-lg-2{padding:0.5rem !important}.pt-lg-2,.py-lg-2{padding-top:0.5rem !important}.pr-lg-2,.px-lg-2{padding-right:0.5rem !important}.pb-lg-2,.py-lg-2{padding-bottom:0.5rem !important}.pl-lg-2,.px-lg-2{padding-left:0.5rem !important}.p-lg-3{padding:1rem !important}.pt-lg-3,.py-lg-3{padding-top:1rem !important}.pr-lg-3,.px-lg-3{padding-right:1rem !important}.pb-lg-3,.py-lg-3{padding-bottom:1rem !important}.pl-lg-3,.px-lg-3{padding-left:1rem !important}.p-lg-4{padding:1.5rem !important}.pt-lg-4,.py-lg-4{padding-top:1.5rem !important}.pr-lg-4,.px-lg-4{padding-right:1.5rem !important}.pb-lg-4,.py-lg-4{padding-bottom:1.5rem !important}.pl-lg-4,.px-lg-4{padding-left:1.5rem !important}.p-lg-5{padding:3rem !important}.pt-lg-5,.py-lg-5{padding-top:3rem !important}.pr-lg-5,.px-lg-5{padding-right:3rem !important}.pb-lg-5,.py-lg-5{padding-bottom:3rem !important}.pl-lg-5,.px-lg-5{padding-left:3rem !important}.m-lg-auto{margin:auto !important}.mt-lg-auto,.my-lg-auto{margin-top:auto !important}.mr-lg-auto,.mx-lg-auto{margin-right:auto !important}.mb-lg-auto,.my-lg-auto{margin-bottom:auto !important}.ml-lg-auto,.mx-lg-auto{margin-left:auto !important}}@media (min-width: 1200px){.m-xl-0{margin:0 !important}.mt-xl-0,.my-xl-0{margin-top:0 !important}.mr-xl-0,.mx-xl-0{margin-right:0 !important}.mb-xl-0,.my-xl-0{margin-bottom:0 !important}.ml-xl-0,.mx-xl-0{margin-left:0 !important}.m-xl-1{margin:0.25rem !important}.mt-xl-1,.my-xl-1{margin-top:0.25rem !important}.mr-xl-1,.mx-xl-1{margin-right:0.25rem !important}.mb-xl-1,.my-xl-1{margin-bottom:0.25rem !important}.ml-xl-1,.mx-xl-1{margin-left:0.25rem !important}.m-xl-2{margin:0.5rem !important}.mt-xl-2,.my-xl-2{margin-top:0.5rem !important}.mr-xl-2,.mx-xl-2{margin-right:0.5rem !important}.mb-xl-2,.my-xl-2{margin-bottom:0.5rem !important}.ml-xl-2,.mx-xl-2{margin-left:0.5rem !important}.m-xl-3{margin:1rem !important}.mt-xl-3,.my-xl-3{margin-top:1rem !important}.mr-xl-3,.mx-xl-3{margin-right:1rem !important}.mb-xl-3,.my-xl-3{margin-bottom:1rem !important}.ml-xl-3,.mx-xl-3{margin-left:1rem !important}.m-xl-4{margin:1.5rem !important}.mt-xl-4,.my-xl-4{margin-top:1.5rem !important}.mr-xl-4,.mx-xl-4{margin-right:1.5rem !important}.mb-xl-4,.my-xl-4{margin-bottom:1.5rem !important}.ml-xl-4,.mx-xl-4{margin-left:1.5rem !important}.m-xl-5{margin:3rem !important}.mt-xl-5,.my-xl-5{margin-top:3rem !important}.mr-xl-5,.mx-xl-5{margin-right:3rem !important}.mb-xl-5,.my-xl-5{margin-bottom:3rem !important}.ml-xl-5,.mx-xl-5{margin-left:3rem !important}.p-xl-0{padding:0 !important}.pt-xl-0,.py-xl-0{padding-top:0 !important}.pr-xl-0,.px-xl-0{padding-right:0 !important}.pb-xl-0,.py-xl-0{padding-bottom:0 !important}.pl-xl-0,.px-xl-0{padding-left:0 !important}.p-xl-1{padding:0.25rem !important}.pt-xl-1,.py-xl-1{padding-top:0.25rem !important}.pr-xl-1,.px-xl-1{padding-right:0.25rem !important}.pb-xl-1,.py-xl-1{padding-bottom:0.25rem !important}.pl-xl-1,.px-xl-1{padding-left:0.25rem !important}.p-xl-2{padding:0.5rem !important}.pt-xl-2,.py-xl-2{padding-top:0.5rem !important}.pr-xl-2,.px-xl-2{padding-right:0.5rem !important}.pb-xl-2,.py-xl-2{padding-bottom:0.5rem !important}.pl-xl-2,.px-xl-2{padding-left:0.5rem !important}.p-xl-3{padding:1rem !important}.pt-xl-3,.py-xl-3{padding-top:1rem !important}.pr-xl-3,.px-xl-3{padding-right:1rem !important}.pb-xl-3,.py-xl-3{padding-bottom:1rem !important}.pl-xl-3,.px-xl-3{padding-left:1rem !important}.p-xl-4{padding:1.5rem !important}.pt-xl-4,.py-xl-4{padding-top:1.5rem !important}.pr-xl-4,.px-xl-4{padding-right:1.5rem !important}.pb-xl-4,.py-xl-4{padding-bottom:1.5rem !important}.pl-xl-4,.px-xl-4{padding-left:1.5rem !important}.p-xl-5{padding:3rem !important}.pt-xl-5,.py-xl-5{padding-top:3rem !important}.pr-xl-5,.px-xl-5{padding-right:3rem !important}.pb-xl-5,.py-xl-5{padding-bottom:3rem !important}.pl-xl-5,.px-xl-5{padding-left:3rem !important}.m-xl-auto{margin:auto !important}.mt-xl-auto,.my-xl-auto{margin-top:auto !important}.mr-xl-auto,.mx-xl-auto{margin-right:auto !important}.mb-xl-auto,.my-xl-auto{margin-bottom:auto !important}.ml-xl-auto,.mx-xl-auto{margin-left:auto !important}}.text-monospace{font-family:SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace}.text-justify{text-align:justify !important}.text-nowrap{white-space:nowrap !important}.text-truncate{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.text-left{text-align:left !important}.text-right{text-align:right !important}.text-center{text-align:center !important}@media (min-width: 576px){.text-sm-left{text-align:left !important}.text-sm-right{text-align:right !important}.text-sm-center{text-align:center !important}}@media (min-width: 768px){.text-md-left{text-align:left !important}.text-md-right{text-align:right !important}.text-md-center{text-align:center !important}}@media (min-width: 992px){.text-lg-left{text-align:left !important}.text-lg-right{text-align:right !important}.text-lg-center{text-align:center !important}}@media (min-width: 1200px){.text-xl-left{text-align:left !important}.text-xl-right{text-align:right !important}.text-xl-center{text-align:center !important}}.text-lowercase{text-transform:lowercase !important}.text-uppercase{text-transform:uppercase !important}.text-capitalize{text-transform:capitalize !important}.font-weight-light{font-weight:300 !important}.font-weight-normal{font-weight:400 !important}.font-weight-bold{font-weight:700 !important}.font-italic{font-style:italic !important}.text-white{color:#fff !important}.text-primary{color:#2FA4E7 !important}a.text-primary:hover,a.text-primary:focus{color:#178acc !important}.text-secondary{color:#e9ecef !important}a.text-secondary:hover,a.text-secondary:focus{color:#cbd3da !important}.text-success{color:#73A839 !important}a.text-success:hover,a.text-success:focus{color:#59822c !important}.text-info{color:#033C73 !important}a.text-info:hover,a.text-info:focus{color:#022241 !important}.text-warning{color:#DD5600 !important}a.text-warning:hover,a.text-warning:focus{color:#aa4200 !important}.text-danger{color:#C71C22 !important}a.text-danger:hover,a.text-danger:focus{color:#9a161a !important}.text-light{color:#f8f9fa !important}a.text-light:hover,a.text-light:focus{color:#dae0e5 !important}.text-dark{color:#343a40 !important}a.text-dark:hover,a.text-dark:focus{color:#1d2124 !important}.text-body{color:#495057 !important}.text-muted{color:#868e96 !important}.text-black-50{color:rgba(0,0,0,0.5) !important}.text-white-50{color:rgba(255,255,255,0.5) !important}.text-hide{font:0/0 a;color:transparent;text-shadow:none;background-color:transparent;border:0}.visible{visibility:visible !important}.invisible{visibility:hidden !important}@media print{*,*::before,*::after{text-shadow:none !important;-webkit-box-shadow:none !important;box-shadow:none !important}a:not(.btn){text-decoration:underline}abbr[title]::after{content:" (" attr(title) ")"}pre{white-space:pre-wrap !important}pre,blockquote{border:1px solid #adb5bd;page-break-inside:avoid}thead{display:table-header-group}tr,img{page-break-inside:avoid}p,h2,h3{orphans:3;widows:3}h2,h3{page-break-after:avoid}@page{size:a3}body{min-width:992px !important}.container{min-width:992px !important}.navbar{display:none}.badge{border:1px solid #000}.table{border-collapse:collapse !important}.table td,.table th{background-color:#fff !important}.table-bordered th,.table-bordered td{border:1px solid #dee2e6 !important}.table-dark{color:inherit}.table-dark th,.table-dark td,.table-dark thead th,.table-dark tbody+tbody{border-color:#dee2e6}.table .thead-dark th{color:inherit;border-color:#dee2e6}}.bg-primary{background-image:-webkit-gradient(linear, left top, left bottom, from(#54b4eb), color-stop(60%, #2FA4E7), to(#1d9ce5));background-image:linear-gradient(#54b4eb, #2FA4E7 60%, #1d9ce5);background-repeat:no-repeat}.bg-dark{background-image:-webkit-gradient(linear, left top, left bottom, from(#04519b), color-stop(60%, #033C73), to(#02325f));background-image:linear-gradient(#04519b, #033C73 60%, #02325f);background-repeat:no-repeat}.bg-light{background-image:-webkit-gradient(linear, left top, left bottom, from(white), color-stop(60%, #e9ecef), to(#e3e7eb));background-image:linear-gradient(white, #e9ecef 60%, #e3e7eb);background-repeat:no-repeat}.navbar-brand,.nav-link{text-shadow:0 1px 0 rgba(0,0,0,0.05)}.btn{text-shadow:0 1px 0 rgba(0,0,0,0.05)}.btn-primary{background-image:-webkit-gradient(linear, left top, left bottom, from(#54b4eb), color-stop(60%, #2FA4E7), to(#1d9ce5));background-image:linear-gradient(#54b4eb, #2FA4E7 60%, #1d9ce5);background-repeat:no-repeat}.btn-secondary{background-image:-webkit-gradient(linear, left top, left bottom, from(white), color-stop(60%, #e9ecef), to(#dde2e6));background-image:linear-gradient(white, #e9ecef 60%, #dde2e6);background-repeat:no-repeat;color:#495057}.btn-success{background-image:-webkit-gradient(linear, left top, left bottom, from(#88c149), color-stop(60%, #73A839), to(#699934));background-image:linear-gradient(#88c149, #73A839 60%, #699934);background-repeat:no-repeat}.btn-info{background-image:-webkit-gradient(linear, left top, left bottom, from(#04519b), color-stop(60%, #033C73), to(#02325f));background-image:linear-gradient(#04519b, #033C73 60%, #02325f);background-repeat:no-repeat}.btn-warning{background-image:-webkit-gradient(linear, left top, left bottom, from(#ff6707), color-stop(60%, #DD5600), to(#c94e00));background-image:linear-gradient(#ff6707, #DD5600 60%, #c94e00);background-repeat:no-repeat}.btn-danger{background-image:-webkit-gradient(linear, left top, left bottom, from(#e12b31), color-stop(60%, #C71C22), to(#b5191f));background-image:linear-gradient(#e12b31, #C71C22 60%, #b5191f);background-repeat:no-repeat}.btn-light{background-image:-webkit-gradient(linear, left top, left bottom, from(white), color-stop(60%, #f8f9fa), to(#eceff2));background-image:linear-gradient(white, #f8f9fa 60%, #eceff2);background-repeat:no-repeat}.btn-dark{background-image:-webkit-gradient(linear, left top, left bottom, from(#464e57), color-stop(60%, #343a40), to(#2b3035));background-image:linear-gradient(#464e57, #343a40 60%, #2b3035);background-repeat:no-repeat}.bg-primary h1,.bg-primary h2,.bg-primary h3,.bg-primary h4,.bg-primary h5,.bg-primary h6,.bg-success h1,.bg-success h2,.bg-success h3,.bg-success h4,.bg-success h5,.bg-success h6,.bg-info h1,.bg-info h2,.bg-info h3,.bg-info h4,.bg-info h5,.bg-info h6,.bg-warning h1,.bg-warning h2,.bg-warning h3,.bg-warning h4,.bg-warning h5,.bg-warning h6,.bg-danger h1,.bg-danger h2,.bg-danger h3,.bg-danger h4,.bg-danger h5,.bg-danger h6,.bg-dark h1,.bg-dark h2,.bg-dark h3,.bg-dark h4,.bg-dark h5,.bg-dark h6{color:#fff}.dropdown-menu .dropdown-header{color:#868e96} diff --git a/docs/site/css/font-awesome.min.css b/docs/site/css/font-awesome.min.css new file mode 100644 index 000000000..540440ce8 --- /dev/null +++ b/docs/site/css/font-awesome.min.css @@ -0,0 +1,4 @@ +/*! + * Font Awesome 4.7.0 by @davegandy - http://fontawesome.io - @fontawesome + * License - http://fontawesome.io/license (Font: SIL OFL 1.1, CSS: MIT License) + */@font-face{font-family:'FontAwesome';src:url('../fonts/fontawesome-webfont.eot?v=4.7.0');src:url('../fonts/fontawesome-webfont.eot?#iefix&v=4.7.0') format('embedded-opentype'),url('../fonts/fontawesome-webfont.woff2?v=4.7.0') format('woff2'),url('../fonts/fontawesome-webfont.woff?v=4.7.0') format('woff'),url('../fonts/fontawesome-webfont.ttf?v=4.7.0') format('truetype'),url('../fonts/fontawesome-webfont.svg?v=4.7.0#fontawesomeregular') format('svg');font-weight:normal;font-style:normal}.fa{display:inline-block;font:normal normal normal 14px/1 FontAwesome;font-size:inherit;text-rendering:auto;-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}.fa-lg{font-size:1.33333333em;line-height:.75em;vertical-align:-15%}.fa-2x{font-size:2em}.fa-3x{font-size:3em}.fa-4x{font-size:4em}.fa-5x{font-size:5em}.fa-fw{width:1.28571429em;text-align:center}.fa-ul{padding-left:0;margin-left:2.14285714em;list-style-type:none}.fa-ul>li{position:relative}.fa-li{position:absolute;left:-2.14285714em;width:2.14285714em;top:.14285714em;text-align:center}.fa-li.fa-lg{left:-1.85714286em}.fa-border{padding:.2em .25em .15em;border:solid .08em #eee;border-radius:.1em}.fa-pull-left{float:left}.fa-pull-right{float:right}.fa.fa-pull-left{margin-right:.3em}.fa.fa-pull-right{margin-left:.3em}.pull-right{float:right}.pull-left{float:left}.fa.pull-left{margin-right:.3em}.fa.pull-right{margin-left:.3em}.fa-spin{-webkit-animation:fa-spin 2s infinite linear;animation:fa-spin 2s infinite linear}.fa-pulse{-webkit-animation:fa-spin 1s infinite steps(8);animation:fa-spin 1s infinite steps(8)}@-webkit-keyframes fa-spin{0%{-webkit-transform:rotate(0deg);transform:rotate(0deg)}100%{-webkit-transform:rotate(359deg);transform:rotate(359deg)}}@keyframes fa-spin{0%{-webkit-transform:rotate(0deg);transform:rotate(0deg)}100%{-webkit-transform:rotate(359deg);transform:rotate(359deg)}}.fa-rotate-90{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=1)";-webkit-transform:rotate(90deg);-ms-transform:rotate(90deg);transform:rotate(90deg)}.fa-rotate-180{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=2)";-webkit-transform:rotate(180deg);-ms-transform:rotate(180deg);transform:rotate(180deg)}.fa-rotate-270{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=3)";-webkit-transform:rotate(270deg);-ms-transform:rotate(270deg);transform:rotate(270deg)}.fa-flip-horizontal{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=0, mirror=1)";-webkit-transform:scale(-1, 1);-ms-transform:scale(-1, 1);transform:scale(-1, 1)}.fa-flip-vertical{-ms-filter:"progid:DXImageTransform.Microsoft.BasicImage(rotation=2, mirror=1)";-webkit-transform:scale(1, -1);-ms-transform:scale(1, -1);transform:scale(1, -1)}:root .fa-rotate-90,:root .fa-rotate-180,:root .fa-rotate-270,:root .fa-flip-horizontal,:root .fa-flip-vertical{filter:none}.fa-stack{position:relative;display:inline-block;width:2em;height:2em;line-height:2em;vertical-align:middle}.fa-stack-1x,.fa-stack-2x{position:absolute;left:0;width:100%;text-align:center}.fa-stack-1x{line-height:inherit}.fa-stack-2x{font-size:2em}.fa-inverse{color:#fff}.fa-glass:before{content:"\f000"}.fa-music:before{content:"\f001"}.fa-search:before{content:"\f002"}.fa-envelope-o:before{content:"\f003"}.fa-heart:before{content:"\f004"}.fa-star:before{content:"\f005"}.fa-star-o:before{content:"\f006"}.fa-user:before{content:"\f007"}.fa-film:before{content:"\f008"}.fa-th-large:before{content:"\f009"}.fa-th:before{content:"\f00a"}.fa-th-list:before{content:"\f00b"}.fa-check:before{content:"\f00c"}.fa-remove:before,.fa-close:before,.fa-times:before{content:"\f00d"}.fa-search-plus:before{content:"\f00e"}.fa-search-minus:before{content:"\f010"}.fa-power-off:before{content:"\f011"}.fa-signal:before{content:"\f012"}.fa-gear:before,.fa-cog:before{content:"\f013"}.fa-trash-o:before{content:"\f014"}.fa-home:before{content:"\f015"}.fa-file-o:before{content:"\f016"}.fa-clock-o:before{content:"\f017"}.fa-road:before{content:"\f018"}.fa-download:before{content:"\f019"}.fa-arrow-circle-o-down:before{content:"\f01a"}.fa-arrow-circle-o-up:before{content:"\f01b"}.fa-inbox:before{content:"\f01c"}.fa-play-circle-o:before{content:"\f01d"}.fa-rotate-right:before,.fa-repeat:before{content:"\f01e"}.fa-refresh:before{content:"\f021"}.fa-list-alt:before{content:"\f022"}.fa-lock:before{content:"\f023"}.fa-flag:before{content:"\f024"}.fa-headphones:before{content:"\f025"}.fa-volume-off:before{content:"\f026"}.fa-volume-down:before{content:"\f027"}.fa-volume-up:before{content:"\f028"}.fa-qrcode:before{content:"\f029"}.fa-barcode:before{content:"\f02a"}.fa-tag:before{content:"\f02b"}.fa-tags:before{content:"\f02c"}.fa-book:before{content:"\f02d"}.fa-bookmark:before{content:"\f02e"}.fa-print:before{content:"\f02f"}.fa-camera:before{content:"\f030"}.fa-font:before{content:"\f031"}.fa-bold:before{content:"\f032"}.fa-italic:before{content:"\f033"}.fa-text-height:before{content:"\f034"}.fa-text-width:before{content:"\f035"}.fa-align-left:before{content:"\f036"}.fa-align-center:before{content:"\f037"}.fa-align-right:before{content:"\f038"}.fa-align-justify:before{content:"\f039"}.fa-list:before{content:"\f03a"}.fa-dedent:before,.fa-outdent:before{content:"\f03b"}.fa-indent:before{content:"\f03c"}.fa-video-camera:before{content:"\f03d"}.fa-photo:before,.fa-image:before,.fa-picture-o:before{content:"\f03e"}.fa-pencil:before{content:"\f040"}.fa-map-marker:before{content:"\f041"}.fa-adjust:before{content:"\f042"}.fa-tint:before{content:"\f043"}.fa-edit:before,.fa-pencil-square-o:before{content:"\f044"}.fa-share-square-o:before{content:"\f045"}.fa-check-square-o:before{content:"\f046"}.fa-arrows:before{content:"\f047"}.fa-step-backward:before{content:"\f048"}.fa-fast-backward:before{content:"\f049"}.fa-backward:before{content:"\f04a"}.fa-play:before{content:"\f04b"}.fa-pause:before{content:"\f04c"}.fa-stop:before{content:"\f04d"}.fa-forward:before{content:"\f04e"}.fa-fast-forward:before{content:"\f050"}.fa-step-forward:before{content:"\f051"}.fa-eject:before{content:"\f052"}.fa-chevron-left:before{content:"\f053"}.fa-chevron-right:before{content:"\f054"}.fa-plus-circle:before{content:"\f055"}.fa-minus-circle:before{content:"\f056"}.fa-times-circle:before{content:"\f057"}.fa-check-circle:before{content:"\f058"}.fa-question-circle:before{content:"\f059"}.fa-info-circle:before{content:"\f05a"}.fa-crosshairs:before{content:"\f05b"}.fa-times-circle-o:before{content:"\f05c"}.fa-check-circle-o:before{content:"\f05d"}.fa-ban:before{content:"\f05e"}.fa-arrow-left:before{content:"\f060"}.fa-arrow-right:before{content:"\f061"}.fa-arrow-up:before{content:"\f062"}.fa-arrow-down:before{content:"\f063"}.fa-mail-forward:before,.fa-share:before{content:"\f064"}.fa-expand:before{content:"\f065"}.fa-compress:before{content:"\f066"}.fa-plus:before{content:"\f067"}.fa-minus:before{content:"\f068"}.fa-asterisk:before{content:"\f069"}.fa-exclamation-circle:before{content:"\f06a"}.fa-gift:before{content:"\f06b"}.fa-leaf:before{content:"\f06c"}.fa-fire:before{content:"\f06d"}.fa-eye:before{content:"\f06e"}.fa-eye-slash:before{content:"\f070"}.fa-warning:before,.fa-exclamation-triangle:before{content:"\f071"}.fa-plane:before{content:"\f072"}.fa-calendar:before{content:"\f073"}.fa-random:before{content:"\f074"}.fa-comment:before{content:"\f075"}.fa-magnet:before{content:"\f076"}.fa-chevron-up:before{content:"\f077"}.fa-chevron-down:before{content:"\f078"}.fa-retweet:before{content:"\f079"}.fa-shopping-cart:before{content:"\f07a"}.fa-folder:before{content:"\f07b"}.fa-folder-open:before{content:"\f07c"}.fa-arrows-v:before{content:"\f07d"}.fa-arrows-h:before{content:"\f07e"}.fa-bar-chart-o:before,.fa-bar-chart:before{content:"\f080"}.fa-twitter-square:before{content:"\f081"}.fa-facebook-square:before{content:"\f082"}.fa-camera-retro:before{content:"\f083"}.fa-key:before{content:"\f084"}.fa-gears:before,.fa-cogs:before{content:"\f085"}.fa-comments:before{content:"\f086"}.fa-thumbs-o-up:before{content:"\f087"}.fa-thumbs-o-down:before{content:"\f088"}.fa-star-half:before{content:"\f089"}.fa-heart-o:before{content:"\f08a"}.fa-sign-out:before{content:"\f08b"}.fa-linkedin-square:before{content:"\f08c"}.fa-thumb-tack:before{content:"\f08d"}.fa-external-link:before{content:"\f08e"}.fa-sign-in:before{content:"\f090"}.fa-trophy:before{content:"\f091"}.fa-github-square:before{content:"\f092"}.fa-upload:before{content:"\f093"}.fa-lemon-o:before{content:"\f094"}.fa-phone:before{content:"\f095"}.fa-square-o:before{content:"\f096"}.fa-bookmark-o:before{content:"\f097"}.fa-phone-square:before{content:"\f098"}.fa-twitter:before{content:"\f099"}.fa-facebook-f:before,.fa-facebook:before{content:"\f09a"}.fa-github:before{content:"\f09b"}.fa-unlock:before{content:"\f09c"}.fa-credit-card:before{content:"\f09d"}.fa-feed:before,.fa-rss:before{content:"\f09e"}.fa-hdd-o:before{content:"\f0a0"}.fa-bullhorn:before{content:"\f0a1"}.fa-bell:before{content:"\f0f3"}.fa-certificate:before{content:"\f0a3"}.fa-hand-o-right:before{content:"\f0a4"}.fa-hand-o-left:before{content:"\f0a5"}.fa-hand-o-up:before{content:"\f0a6"}.fa-hand-o-down:before{content:"\f0a7"}.fa-arrow-circle-left:before{content:"\f0a8"}.fa-arrow-circle-right:before{content:"\f0a9"}.fa-arrow-circle-up:before{content:"\f0aa"}.fa-arrow-circle-down:before{content:"\f0ab"}.fa-globe:before{content:"\f0ac"}.fa-wrench:before{content:"\f0ad"}.fa-tasks:before{content:"\f0ae"}.fa-filter:before{content:"\f0b0"}.fa-briefcase:before{content:"\f0b1"}.fa-arrows-alt:before{content:"\f0b2"}.fa-group:before,.fa-users:before{content:"\f0c0"}.fa-chain:before,.fa-link:before{content:"\f0c1"}.fa-cloud:before{content:"\f0c2"}.fa-flask:before{content:"\f0c3"}.fa-cut:before,.fa-scissors:before{content:"\f0c4"}.fa-copy:before,.fa-files-o:before{content:"\f0c5"}.fa-paperclip:before{content:"\f0c6"}.fa-save:before,.fa-floppy-o:before{content:"\f0c7"}.fa-square:before{content:"\f0c8"}.fa-navicon:before,.fa-reorder:before,.fa-bars:before{content:"\f0c9"}.fa-list-ul:before{content:"\f0ca"}.fa-list-ol:before{content:"\f0cb"}.fa-strikethrough:before{content:"\f0cc"}.fa-underline:before{content:"\f0cd"}.fa-table:before{content:"\f0ce"}.fa-magic:before{content:"\f0d0"}.fa-truck:before{content:"\f0d1"}.fa-pinterest:before{content:"\f0d2"}.fa-pinterest-square:before{content:"\f0d3"}.fa-google-plus-square:before{content:"\f0d4"}.fa-google-plus:before{content:"\f0d5"}.fa-money:before{content:"\f0d6"}.fa-caret-down:before{content:"\f0d7"}.fa-caret-up:before{content:"\f0d8"}.fa-caret-left:before{content:"\f0d9"}.fa-caret-right:before{content:"\f0da"}.fa-columns:before{content:"\f0db"}.fa-unsorted:before,.fa-sort:before{content:"\f0dc"}.fa-sort-down:before,.fa-sort-desc:before{content:"\f0dd"}.fa-sort-up:before,.fa-sort-asc:before{content:"\f0de"}.fa-envelope:before{content:"\f0e0"}.fa-linkedin:before{content:"\f0e1"}.fa-rotate-left:before,.fa-undo:before{content:"\f0e2"}.fa-legal:before,.fa-gavel:before{content:"\f0e3"}.fa-dashboard:before,.fa-tachometer:before{content:"\f0e4"}.fa-comment-o:before{content:"\f0e5"}.fa-comments-o:before{content:"\f0e6"}.fa-flash:before,.fa-bolt:before{content:"\f0e7"}.fa-sitemap:before{content:"\f0e8"}.fa-umbrella:before{content:"\f0e9"}.fa-paste:before,.fa-clipboard:before{content:"\f0ea"}.fa-lightbulb-o:before{content:"\f0eb"}.fa-exchange:before{content:"\f0ec"}.fa-cloud-download:before{content:"\f0ed"}.fa-cloud-upload:before{content:"\f0ee"}.fa-user-md:before{content:"\f0f0"}.fa-stethoscope:before{content:"\f0f1"}.fa-suitcase:before{content:"\f0f2"}.fa-bell-o:before{content:"\f0a2"}.fa-coffee:before{content:"\f0f4"}.fa-cutlery:before{content:"\f0f5"}.fa-file-text-o:before{content:"\f0f6"}.fa-building-o:before{content:"\f0f7"}.fa-hospital-o:before{content:"\f0f8"}.fa-ambulance:before{content:"\f0f9"}.fa-medkit:before{content:"\f0fa"}.fa-fighter-jet:before{content:"\f0fb"}.fa-beer:before{content:"\f0fc"}.fa-h-square:before{content:"\f0fd"}.fa-plus-square:before{content:"\f0fe"}.fa-angle-double-left:before{content:"\f100"}.fa-angle-double-right:before{content:"\f101"}.fa-angle-double-up:before{content:"\f102"}.fa-angle-double-down:before{content:"\f103"}.fa-angle-left:before{content:"\f104"}.fa-angle-right:before{content:"\f105"}.fa-angle-up:before{content:"\f106"}.fa-angle-down:before{content:"\f107"}.fa-desktop:before{content:"\f108"}.fa-laptop:before{content:"\f109"}.fa-tablet:before{content:"\f10a"}.fa-mobile-phone:before,.fa-mobile:before{content:"\f10b"}.fa-circle-o:before{content:"\f10c"}.fa-quote-left:before{content:"\f10d"}.fa-quote-right:before{content:"\f10e"}.fa-spinner:before{content:"\f110"}.fa-circle:before{content:"\f111"}.fa-mail-reply:before,.fa-reply:before{content:"\f112"}.fa-github-alt:before{content:"\f113"}.fa-folder-o:before{content:"\f114"}.fa-folder-open-o:before{content:"\f115"}.fa-smile-o:before{content:"\f118"}.fa-frown-o:before{content:"\f119"}.fa-meh-o:before{content:"\f11a"}.fa-gamepad:before{content:"\f11b"}.fa-keyboard-o:before{content:"\f11c"}.fa-flag-o:before{content:"\f11d"}.fa-flag-checkered:before{content:"\f11e"}.fa-terminal:before{content:"\f120"}.fa-code:before{content:"\f121"}.fa-mail-reply-all:before,.fa-reply-all:before{content:"\f122"}.fa-star-half-empty:before,.fa-star-half-full:before,.fa-star-half-o:before{content:"\f123"}.fa-location-arrow:before{content:"\f124"}.fa-crop:before{content:"\f125"}.fa-code-fork:before{content:"\f126"}.fa-unlink:before,.fa-chain-broken:before{content:"\f127"}.fa-question:before{content:"\f128"}.fa-info:before{content:"\f129"}.fa-exclamation:before{content:"\f12a"}.fa-superscript:before{content:"\f12b"}.fa-subscript:before{content:"\f12c"}.fa-eraser:before{content:"\f12d"}.fa-puzzle-piece:before{content:"\f12e"}.fa-microphone:before{content:"\f130"}.fa-microphone-slash:before{content:"\f131"}.fa-shield:before{content:"\f132"}.fa-calendar-o:before{content:"\f133"}.fa-fire-extinguisher:before{content:"\f134"}.fa-rocket:before{content:"\f135"}.fa-maxcdn:before{content:"\f136"}.fa-chevron-circle-left:before{content:"\f137"}.fa-chevron-circle-right:before{content:"\f138"}.fa-chevron-circle-up:before{content:"\f139"}.fa-chevron-circle-down:before{content:"\f13a"}.fa-html5:before{content:"\f13b"}.fa-css3:before{content:"\f13c"}.fa-anchor:before{content:"\f13d"}.fa-unlock-alt:before{content:"\f13e"}.fa-bullseye:before{content:"\f140"}.fa-ellipsis-h:before{content:"\f141"}.fa-ellipsis-v:before{content:"\f142"}.fa-rss-square:before{content:"\f143"}.fa-play-circle:before{content:"\f144"}.fa-ticket:before{content:"\f145"}.fa-minus-square:before{content:"\f146"}.fa-minus-square-o:before{content:"\f147"}.fa-level-up:before{content:"\f148"}.fa-level-down:before{content:"\f149"}.fa-check-square:before{content:"\f14a"}.fa-pencil-square:before{content:"\f14b"}.fa-external-link-square:before{content:"\f14c"}.fa-share-square:before{content:"\f14d"}.fa-compass:before{content:"\f14e"}.fa-toggle-down:before,.fa-caret-square-o-down:before{content:"\f150"}.fa-toggle-up:before,.fa-caret-square-o-up:before{content:"\f151"}.fa-toggle-right:before,.fa-caret-square-o-right:before{content:"\f152"}.fa-euro:before,.fa-eur:before{content:"\f153"}.fa-gbp:before{content:"\f154"}.fa-dollar:before,.fa-usd:before{content:"\f155"}.fa-rupee:before,.fa-inr:before{content:"\f156"}.fa-cny:before,.fa-rmb:before,.fa-yen:before,.fa-jpy:before{content:"\f157"}.fa-ruble:before,.fa-rouble:before,.fa-rub:before{content:"\f158"}.fa-won:before,.fa-krw:before{content:"\f159"}.fa-bitcoin:before,.fa-btc:before{content:"\f15a"}.fa-file:before{content:"\f15b"}.fa-file-text:before{content:"\f15c"}.fa-sort-alpha-asc:before{content:"\f15d"}.fa-sort-alpha-desc:before{content:"\f15e"}.fa-sort-amount-asc:before{content:"\f160"}.fa-sort-amount-desc:before{content:"\f161"}.fa-sort-numeric-asc:before{content:"\f162"}.fa-sort-numeric-desc:before{content:"\f163"}.fa-thumbs-up:before{content:"\f164"}.fa-thumbs-down:before{content:"\f165"}.fa-youtube-square:before{content:"\f166"}.fa-youtube:before{content:"\f167"}.fa-xing:before{content:"\f168"}.fa-xing-square:before{content:"\f169"}.fa-youtube-play:before{content:"\f16a"}.fa-dropbox:before{content:"\f16b"}.fa-stack-overflow:before{content:"\f16c"}.fa-instagram:before{content:"\f16d"}.fa-flickr:before{content:"\f16e"}.fa-adn:before{content:"\f170"}.fa-bitbucket:before{content:"\f171"}.fa-bitbucket-square:before{content:"\f172"}.fa-tumblr:before{content:"\f173"}.fa-tumblr-square:before{content:"\f174"}.fa-long-arrow-down:before{content:"\f175"}.fa-long-arrow-up:before{content:"\f176"}.fa-long-arrow-left:before{content:"\f177"}.fa-long-arrow-right:before{content:"\f178"}.fa-apple:before{content:"\f179"}.fa-windows:before{content:"\f17a"}.fa-android:before{content:"\f17b"}.fa-linux:before{content:"\f17c"}.fa-dribbble:before{content:"\f17d"}.fa-skype:before{content:"\f17e"}.fa-foursquare:before{content:"\f180"}.fa-trello:before{content:"\f181"}.fa-female:before{content:"\f182"}.fa-male:before{content:"\f183"}.fa-gittip:before,.fa-gratipay:before{content:"\f184"}.fa-sun-o:before{content:"\f185"}.fa-moon-o:before{content:"\f186"}.fa-archive:before{content:"\f187"}.fa-bug:before{content:"\f188"}.fa-vk:before{content:"\f189"}.fa-weibo:before{content:"\f18a"}.fa-renren:before{content:"\f18b"}.fa-pagelines:before{content:"\f18c"}.fa-stack-exchange:before{content:"\f18d"}.fa-arrow-circle-o-right:before{content:"\f18e"}.fa-arrow-circle-o-left:before{content:"\f190"}.fa-toggle-left:before,.fa-caret-square-o-left:before{content:"\f191"}.fa-dot-circle-o:before{content:"\f192"}.fa-wheelchair:before{content:"\f193"}.fa-vimeo-square:before{content:"\f194"}.fa-turkish-lira:before,.fa-try:before{content:"\f195"}.fa-plus-square-o:before{content:"\f196"}.fa-space-shuttle:before{content:"\f197"}.fa-slack:before{content:"\f198"}.fa-envelope-square:before{content:"\f199"}.fa-wordpress:before{content:"\f19a"}.fa-openid:before{content:"\f19b"}.fa-institution:before,.fa-bank:before,.fa-university:before{content:"\f19c"}.fa-mortar-board:before,.fa-graduation-cap:before{content:"\f19d"}.fa-yahoo:before{content:"\f19e"}.fa-google:before{content:"\f1a0"}.fa-reddit:before{content:"\f1a1"}.fa-reddit-square:before{content:"\f1a2"}.fa-stumbleupon-circle:before{content:"\f1a3"}.fa-stumbleupon:before{content:"\f1a4"}.fa-delicious:before{content:"\f1a5"}.fa-digg:before{content:"\f1a6"}.fa-pied-piper-pp:before{content:"\f1a7"}.fa-pied-piper-alt:before{content:"\f1a8"}.fa-drupal:before{content:"\f1a9"}.fa-joomla:before{content:"\f1aa"}.fa-language:before{content:"\f1ab"}.fa-fax:before{content:"\f1ac"}.fa-building:before{content:"\f1ad"}.fa-child:before{content:"\f1ae"}.fa-paw:before{content:"\f1b0"}.fa-spoon:before{content:"\f1b1"}.fa-cube:before{content:"\f1b2"}.fa-cubes:before{content:"\f1b3"}.fa-behance:before{content:"\f1b4"}.fa-behance-square:before{content:"\f1b5"}.fa-steam:before{content:"\f1b6"}.fa-steam-square:before{content:"\f1b7"}.fa-recycle:before{content:"\f1b8"}.fa-automobile:before,.fa-car:before{content:"\f1b9"}.fa-cab:before,.fa-taxi:before{content:"\f1ba"}.fa-tree:before{content:"\f1bb"}.fa-spotify:before{content:"\f1bc"}.fa-deviantart:before{content:"\f1bd"}.fa-soundcloud:before{content:"\f1be"}.fa-database:before{content:"\f1c0"}.fa-file-pdf-o:before{content:"\f1c1"}.fa-file-word-o:before{content:"\f1c2"}.fa-file-excel-o:before{content:"\f1c3"}.fa-file-powerpoint-o:before{content:"\f1c4"}.fa-file-photo-o:before,.fa-file-picture-o:before,.fa-file-image-o:before{content:"\f1c5"}.fa-file-zip-o:before,.fa-file-archive-o:before{content:"\f1c6"}.fa-file-sound-o:before,.fa-file-audio-o:before{content:"\f1c7"}.fa-file-movie-o:before,.fa-file-video-o:before{content:"\f1c8"}.fa-file-code-o:before{content:"\f1c9"}.fa-vine:before{content:"\f1ca"}.fa-codepen:before{content:"\f1cb"}.fa-jsfiddle:before{content:"\f1cc"}.fa-life-bouy:before,.fa-life-buoy:before,.fa-life-saver:before,.fa-support:before,.fa-life-ring:before{content:"\f1cd"}.fa-circle-o-notch:before{content:"\f1ce"}.fa-ra:before,.fa-resistance:before,.fa-rebel:before{content:"\f1d0"}.fa-ge:before,.fa-empire:before{content:"\f1d1"}.fa-git-square:before{content:"\f1d2"}.fa-git:before{content:"\f1d3"}.fa-y-combinator-square:before,.fa-yc-square:before,.fa-hacker-news:before{content:"\f1d4"}.fa-tencent-weibo:before{content:"\f1d5"}.fa-qq:before{content:"\f1d6"}.fa-wechat:before,.fa-weixin:before{content:"\f1d7"}.fa-send:before,.fa-paper-plane:before{content:"\f1d8"}.fa-send-o:before,.fa-paper-plane-o:before{content:"\f1d9"}.fa-history:before{content:"\f1da"}.fa-circle-thin:before{content:"\f1db"}.fa-header:before{content:"\f1dc"}.fa-paragraph:before{content:"\f1dd"}.fa-sliders:before{content:"\f1de"}.fa-share-alt:before{content:"\f1e0"}.fa-share-alt-square:before{content:"\f1e1"}.fa-bomb:before{content:"\f1e2"}.fa-soccer-ball-o:before,.fa-futbol-o:before{content:"\f1e3"}.fa-tty:before{content:"\f1e4"}.fa-binoculars:before{content:"\f1e5"}.fa-plug:before{content:"\f1e6"}.fa-slideshare:before{content:"\f1e7"}.fa-twitch:before{content:"\f1e8"}.fa-yelp:before{content:"\f1e9"}.fa-newspaper-o:before{content:"\f1ea"}.fa-wifi:before{content:"\f1eb"}.fa-calculator:before{content:"\f1ec"}.fa-paypal:before{content:"\f1ed"}.fa-google-wallet:before{content:"\f1ee"}.fa-cc-visa:before{content:"\f1f0"}.fa-cc-mastercard:before{content:"\f1f1"}.fa-cc-discover:before{content:"\f1f2"}.fa-cc-amex:before{content:"\f1f3"}.fa-cc-paypal:before{content:"\f1f4"}.fa-cc-stripe:before{content:"\f1f5"}.fa-bell-slash:before{content:"\f1f6"}.fa-bell-slash-o:before{content:"\f1f7"}.fa-trash:before{content:"\f1f8"}.fa-copyright:before{content:"\f1f9"}.fa-at:before{content:"\f1fa"}.fa-eyedropper:before{content:"\f1fb"}.fa-paint-brush:before{content:"\f1fc"}.fa-birthday-cake:before{content:"\f1fd"}.fa-area-chart:before{content:"\f1fe"}.fa-pie-chart:before{content:"\f200"}.fa-line-chart:before{content:"\f201"}.fa-lastfm:before{content:"\f202"}.fa-lastfm-square:before{content:"\f203"}.fa-toggle-off:before{content:"\f204"}.fa-toggle-on:before{content:"\f205"}.fa-bicycle:before{content:"\f206"}.fa-bus:before{content:"\f207"}.fa-ioxhost:before{content:"\f208"}.fa-angellist:before{content:"\f209"}.fa-cc:before{content:"\f20a"}.fa-shekel:before,.fa-sheqel:before,.fa-ils:before{content:"\f20b"}.fa-meanpath:before{content:"\f20c"}.fa-buysellads:before{content:"\f20d"}.fa-connectdevelop:before{content:"\f20e"}.fa-dashcube:before{content:"\f210"}.fa-forumbee:before{content:"\f211"}.fa-leanpub:before{content:"\f212"}.fa-sellsy:before{content:"\f213"}.fa-shirtsinbulk:before{content:"\f214"}.fa-simplybuilt:before{content:"\f215"}.fa-skyatlas:before{content:"\f216"}.fa-cart-plus:before{content:"\f217"}.fa-cart-arrow-down:before{content:"\f218"}.fa-diamond:before{content:"\f219"}.fa-ship:before{content:"\f21a"}.fa-user-secret:before{content:"\f21b"}.fa-motorcycle:before{content:"\f21c"}.fa-street-view:before{content:"\f21d"}.fa-heartbeat:before{content:"\f21e"}.fa-venus:before{content:"\f221"}.fa-mars:before{content:"\f222"}.fa-mercury:before{content:"\f223"}.fa-intersex:before,.fa-transgender:before{content:"\f224"}.fa-transgender-alt:before{content:"\f225"}.fa-venus-double:before{content:"\f226"}.fa-mars-double:before{content:"\f227"}.fa-venus-mars:before{content:"\f228"}.fa-mars-stroke:before{content:"\f229"}.fa-mars-stroke-v:before{content:"\f22a"}.fa-mars-stroke-h:before{content:"\f22b"}.fa-neuter:before{content:"\f22c"}.fa-genderless:before{content:"\f22d"}.fa-facebook-official:before{content:"\f230"}.fa-pinterest-p:before{content:"\f231"}.fa-whatsapp:before{content:"\f232"}.fa-server:before{content:"\f233"}.fa-user-plus:before{content:"\f234"}.fa-user-times:before{content:"\f235"}.fa-hotel:before,.fa-bed:before{content:"\f236"}.fa-viacoin:before{content:"\f237"}.fa-train:before{content:"\f238"}.fa-subway:before{content:"\f239"}.fa-medium:before{content:"\f23a"}.fa-yc:before,.fa-y-combinator:before{content:"\f23b"}.fa-optin-monster:before{content:"\f23c"}.fa-opencart:before{content:"\f23d"}.fa-expeditedssl:before{content:"\f23e"}.fa-battery-4:before,.fa-battery:before,.fa-battery-full:before{content:"\f240"}.fa-battery-3:before,.fa-battery-three-quarters:before{content:"\f241"}.fa-battery-2:before,.fa-battery-half:before{content:"\f242"}.fa-battery-1:before,.fa-battery-quarter:before{content:"\f243"}.fa-battery-0:before,.fa-battery-empty:before{content:"\f244"}.fa-mouse-pointer:before{content:"\f245"}.fa-i-cursor:before{content:"\f246"}.fa-object-group:before{content:"\f247"}.fa-object-ungroup:before{content:"\f248"}.fa-sticky-note:before{content:"\f249"}.fa-sticky-note-o:before{content:"\f24a"}.fa-cc-jcb:before{content:"\f24b"}.fa-cc-diners-club:before{content:"\f24c"}.fa-clone:before{content:"\f24d"}.fa-balance-scale:before{content:"\f24e"}.fa-hourglass-o:before{content:"\f250"}.fa-hourglass-1:before,.fa-hourglass-start:before{content:"\f251"}.fa-hourglass-2:before,.fa-hourglass-half:before{content:"\f252"}.fa-hourglass-3:before,.fa-hourglass-end:before{content:"\f253"}.fa-hourglass:before{content:"\f254"}.fa-hand-grab-o:before,.fa-hand-rock-o:before{content:"\f255"}.fa-hand-stop-o:before,.fa-hand-paper-o:before{content:"\f256"}.fa-hand-scissors-o:before{content:"\f257"}.fa-hand-lizard-o:before{content:"\f258"}.fa-hand-spock-o:before{content:"\f259"}.fa-hand-pointer-o:before{content:"\f25a"}.fa-hand-peace-o:before{content:"\f25b"}.fa-trademark:before{content:"\f25c"}.fa-registered:before{content:"\f25d"}.fa-creative-commons:before{content:"\f25e"}.fa-gg:before{content:"\f260"}.fa-gg-circle:before{content:"\f261"}.fa-tripadvisor:before{content:"\f262"}.fa-odnoklassniki:before{content:"\f263"}.fa-odnoklassniki-square:before{content:"\f264"}.fa-get-pocket:before{content:"\f265"}.fa-wikipedia-w:before{content:"\f266"}.fa-safari:before{content:"\f267"}.fa-chrome:before{content:"\f268"}.fa-firefox:before{content:"\f269"}.fa-opera:before{content:"\f26a"}.fa-internet-explorer:before{content:"\f26b"}.fa-tv:before,.fa-television:before{content:"\f26c"}.fa-contao:before{content:"\f26d"}.fa-500px:before{content:"\f26e"}.fa-amazon:before{content:"\f270"}.fa-calendar-plus-o:before{content:"\f271"}.fa-calendar-minus-o:before{content:"\f272"}.fa-calendar-times-o:before{content:"\f273"}.fa-calendar-check-o:before{content:"\f274"}.fa-industry:before{content:"\f275"}.fa-map-pin:before{content:"\f276"}.fa-map-signs:before{content:"\f277"}.fa-map-o:before{content:"\f278"}.fa-map:before{content:"\f279"}.fa-commenting:before{content:"\f27a"}.fa-commenting-o:before{content:"\f27b"}.fa-houzz:before{content:"\f27c"}.fa-vimeo:before{content:"\f27d"}.fa-black-tie:before{content:"\f27e"}.fa-fonticons:before{content:"\f280"}.fa-reddit-alien:before{content:"\f281"}.fa-edge:before{content:"\f282"}.fa-credit-card-alt:before{content:"\f283"}.fa-codiepie:before{content:"\f284"}.fa-modx:before{content:"\f285"}.fa-fort-awesome:before{content:"\f286"}.fa-usb:before{content:"\f287"}.fa-product-hunt:before{content:"\f288"}.fa-mixcloud:before{content:"\f289"}.fa-scribd:before{content:"\f28a"}.fa-pause-circle:before{content:"\f28b"}.fa-pause-circle-o:before{content:"\f28c"}.fa-stop-circle:before{content:"\f28d"}.fa-stop-circle-o:before{content:"\f28e"}.fa-shopping-bag:before{content:"\f290"}.fa-shopping-basket:before{content:"\f291"}.fa-hashtag:before{content:"\f292"}.fa-bluetooth:before{content:"\f293"}.fa-bluetooth-b:before{content:"\f294"}.fa-percent:before{content:"\f295"}.fa-gitlab:before{content:"\f296"}.fa-wpbeginner:before{content:"\f297"}.fa-wpforms:before{content:"\f298"}.fa-envira:before{content:"\f299"}.fa-universal-access:before{content:"\f29a"}.fa-wheelchair-alt:before{content:"\f29b"}.fa-question-circle-o:before{content:"\f29c"}.fa-blind:before{content:"\f29d"}.fa-audio-description:before{content:"\f29e"}.fa-volume-control-phone:before{content:"\f2a0"}.fa-braille:before{content:"\f2a1"}.fa-assistive-listening-systems:before{content:"\f2a2"}.fa-asl-interpreting:before,.fa-american-sign-language-interpreting:before{content:"\f2a3"}.fa-deafness:before,.fa-hard-of-hearing:before,.fa-deaf:before{content:"\f2a4"}.fa-glide:before{content:"\f2a5"}.fa-glide-g:before{content:"\f2a6"}.fa-signing:before,.fa-sign-language:before{content:"\f2a7"}.fa-low-vision:before{content:"\f2a8"}.fa-viadeo:before{content:"\f2a9"}.fa-viadeo-square:before{content:"\f2aa"}.fa-snapchat:before{content:"\f2ab"}.fa-snapchat-ghost:before{content:"\f2ac"}.fa-snapchat-square:before{content:"\f2ad"}.fa-pied-piper:before{content:"\f2ae"}.fa-first-order:before{content:"\f2b0"}.fa-yoast:before{content:"\f2b1"}.fa-themeisle:before{content:"\f2b2"}.fa-google-plus-circle:before,.fa-google-plus-official:before{content:"\f2b3"}.fa-fa:before,.fa-font-awesome:before{content:"\f2b4"}.fa-handshake-o:before{content:"\f2b5"}.fa-envelope-open:before{content:"\f2b6"}.fa-envelope-open-o:before{content:"\f2b7"}.fa-linode:before{content:"\f2b8"}.fa-address-book:before{content:"\f2b9"}.fa-address-book-o:before{content:"\f2ba"}.fa-vcard:before,.fa-address-card:before{content:"\f2bb"}.fa-vcard-o:before,.fa-address-card-o:before{content:"\f2bc"}.fa-user-circle:before{content:"\f2bd"}.fa-user-circle-o:before{content:"\f2be"}.fa-user-o:before{content:"\f2c0"}.fa-id-badge:before{content:"\f2c1"}.fa-drivers-license:before,.fa-id-card:before{content:"\f2c2"}.fa-drivers-license-o:before,.fa-id-card-o:before{content:"\f2c3"}.fa-quora:before{content:"\f2c4"}.fa-free-code-camp:before{content:"\f2c5"}.fa-telegram:before{content:"\f2c6"}.fa-thermometer-4:before,.fa-thermometer:before,.fa-thermometer-full:before{content:"\f2c7"}.fa-thermometer-3:before,.fa-thermometer-three-quarters:before{content:"\f2c8"}.fa-thermometer-2:before,.fa-thermometer-half:before{content:"\f2c9"}.fa-thermometer-1:before,.fa-thermometer-quarter:before{content:"\f2ca"}.fa-thermometer-0:before,.fa-thermometer-empty:before{content:"\f2cb"}.fa-shower:before{content:"\f2cc"}.fa-bathtub:before,.fa-s15:before,.fa-bath:before{content:"\f2cd"}.fa-podcast:before{content:"\f2ce"}.fa-window-maximize:before{content:"\f2d0"}.fa-window-minimize:before{content:"\f2d1"}.fa-window-restore:before{content:"\f2d2"}.fa-times-rectangle:before,.fa-window-close:before{content:"\f2d3"}.fa-times-rectangle-o:before,.fa-window-close-o:before{content:"\f2d4"}.fa-bandcamp:before{content:"\f2d5"}.fa-grav:before{content:"\f2d6"}.fa-etsy:before{content:"\f2d7"}.fa-imdb:before{content:"\f2d8"}.fa-ravelry:before{content:"\f2d9"}.fa-eercast:before{content:"\f2da"}.fa-microchip:before{content:"\f2db"}.fa-snowflake-o:before{content:"\f2dc"}.fa-superpowers:before{content:"\f2dd"}.fa-wpexplorer:before{content:"\f2de"}.fa-meetup:before{content:"\f2e0"}.sr-only{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0, 0, 0, 0);border:0}.sr-only-focusable:active,.sr-only-focusable:focus{position:static;width:auto;height:auto;margin:0;overflow:visible;clip:auto} diff --git a/docs/site/developer/diagrams/reconciliation-module.png b/docs/site/developer/diagrams/reconciliation-module.png new file mode 100644 index 000000000..d17c21de2 Binary files /dev/null and b/docs/site/developer/diagrams/reconciliation-module.png differ diff --git a/docs/site/developer/index.html b/docs/site/developer/index.html new file mode 100644 index 000000000..5e90bba9e --- /dev/null +++ b/docs/site/developer/index.html @@ -0,0 +1,264 @@ + + + + + + + + + + + Design notes - Out-Of-Tree operator + + + + + + + + + + + + +
+
+ Out-Of-Tree operator + + + + + +
+
+ +
+
+
+
+ +

Design notes

+

Kernel Module Management brings a new Module CRD. +Module represents an out-of-tree kernel module that must be inserted into the cluster nodes’ kernel through a +DriverContainer scheduled by a DaemonSet.

+

KMM optionally builds and runs DriverContainers on the cluster. +It picks the right DriverContainer image by leveraging kernel mappings in the Module CRD that describe which image +should be used for which kernel.

+

A kernel mapping maps either a literal kernel name or a regex with a kernel’s node. +This allows for more flexibility when targeting a set of kernels (e.g. “for Ubuntu nodes, build from that repository”). +Variables available at build time still reflect the actual kernel version.

+

Module CRD

+
apiVersion: ooto.sigs.k8s.io/v1alpha1
+kind: Module
+metadata:
+  name: module-sample
+spec:
+  additionalVolumes:
+    # All volumes here are available into the DriverContainer / DevicePlugin pod.
+    # Users should mount those by adding corresponding volumeMounts under .spec.devicePlugin
+    # and .spec.moduleLoader.
+    - name: some-volume
+      configMap:
+        name: some-configmap
+  devicePlugin: # is a Container spec
+    # This container will be privileged and will mount
+    # /var/lib/kubelet/device-plugins automatically.
+    name: overwritten-anyway
+    image: some-image
+  moduleLoader: # is a Container spec
+    # This container will not be privileged by default.
+    # It will mount /lib/modules and /usr/lib/modules automatically.
+    name: overwritten-anyway
+    image: overwritten-anyway
+    securityContext:
+      capabilities:
+        add: [SYS_MODULE] # this is enough in most cases
+  kernelMappings:
+    - literal: 5.16.11-200.fc35.x86_64
+      containerImage: quay.io/vendor/module-sample:fedora-5.16.11-200.fc35.x86_64
+    - literal: 5.4.0-1054-gke
+      containerImage: quay.io/vendor/module-sample:ubuntu-5.4.0-1054-gke
+    - regexp: '^.*\-gke$'
+      build:
+        buildArgs:
+          - name: SOME_KEY
+            value: SOME_VALUE
+        pull:
+          insecure: false
+        push:
+          insecure: false
+        dockerfile: |
+          FROM some-image
+          RUN some-command
+      containerImage: quay.io/vendor/module-sample:gke
+  selector:  # top-level selector
+    feature.node.kubernetes.io/cpu-cpuid.VMX: true
+status:
+  conditions:
+    - lastTransitionTime: "2022-03-07T14:01:00Z"
+      status: False
+      type: Progressing
+    - lastTransitionTime: "2022-03-07T14:01:01Z"
+      status: True
+      type: Ready
+
+

In-cluster builds

+

Virtually any Kubernetes distribution ships with its own kernel. +It is thus challenging for a kernel module vendor to make DriverContainer images available for all kernels available +in all Kubernetes distributions out there (or even for a subset of them).

+

KMM supports in-cluster builds of DriverContainer images when those are not made available by the vendor. +The in-cluster build system is able to build an image from any Git repository that contains a Dockerfile. +It can then push said image to a user-provided repository, optionally using authentication provided in a Kubernetes secret.

+

Optional: on upstream Kubernetes, we may want to deploy an in-cluster registry to host in-cluster built images.
+On OCP, the build mechanism would be BuildConfig (maybe Shipwright in the future) and we can leverage the +integrated in-cluster registry.

+

Unloading modules

+

Users must provide a preStop lifecycle hook +to their DriverContainer pod template to make sure that their module is unloaded when the DriverContainer pod exits.

+

Security

+

DriverContainer privileges

+

By default, the operator would only grant the CAP_SYS_MODULE +capability to DriverContainer DaemonSets.

+

Kubernetes API privileges

+

The KMM Operator would only be granted a limited set of Kubernetes API privileges:

+
    +
  • Read, modify (for kernel version labeling) and watch Nodes;
    • +
  • Read and watch Modules, update their status;
    • +
  • Read, create, modify and watch DaemonSets;
    • +
  • Read, create, modify and watch Build objects (from whatever system we agree on);
    • +
  • Read Secrets for pull and build secrets.
    • +
+
+
+ +
+
+

Documentation built with MkDocs.

+
+ + + + + + + diff --git a/docs/site/developer/reconciliation_loops/index.html b/docs/site/developer/reconciliation_loops/index.html new file mode 100644 index 000000000..8bff7838a --- /dev/null +++ b/docs/site/developer/reconciliation_loops/index.html @@ -0,0 +1,176 @@ + + + + + + + + + + + Reconciliation loop - Out-Of-Tree operator + + + + + + + + + + + + +
+
+ Out-Of-Tree operator + + + + + +
+
+ +
+
+
+
+ +

Reconciliation loop

+

Modules

+

Each time a new Module is created, we need to find to which nodes it applies.
+A first filtering is performed using the .spec.selector field, then we go through the module’s kernel mappings to +find a container image that matches the node’s kernel. +We end up with a certain number of (kernel, image) pairs; for each of these pairs, there should be a DaemonSet.
+We first look for a DaemonSet that would already be targeting the same kernel and DriverContainer image (that data is +stored in the DaemonSet’s labels). +If there is already such a DaemonSet, we patch it, if needed.
+If there is not already a matching DaemonSet, we create it and set the Module as owner.

+

When a Module is deleted, we have nothing to do: because we set it as owner of all DaemonSets, Kubernetes garbage +collection will take care of deleting them.

+

We watch Module, owned DaemonSet and build objects as well as nodes to make sure that we are not missing any change +in the cluster.

+

Modules reconciliation

+
+
+ +
+
+

Documentation built with MkDocs.

+
+ + + + + + + diff --git a/docs/site/developer/use_cases/index.html b/docs/site/developer/use_cases/index.html new file mode 100644 index 000000000..a4585795d --- /dev/null +++ b/docs/site/developer/use_cases/index.html @@ -0,0 +1,231 @@ + + + + + + + + + + + Use cases - Out-Of-Tree operator + + + + + + + + + + + + +
+
+ Out-Of-Tree operator + + + + + +
+
+ +
+
+
+
+ +

Use cases

+

A new module is added to the cluster

+

The operator lists all nodes required to build the module and looks at their kernels. +It uses the Module’s .spec.kernelMappings to create DaemonSets for each of those kernels.

+

If an image for one of these DaemonSets requires an in-cluster build, it creates the build object (build system TBD). +The Module owns the build object, so that it is deleted when the Module is deleted.

+

Module update: kernel $kernel should run a new DriverContainer image

+

The OOTO would be triggered by an update to any Module instance. +That reconciliation generates a list of DaemonSets that need to be reconciled.

+

For each DaemonSet belonging to Module, we set the following labels:

+ + + + + + + + + + + + + + + + + +
keyvalue
ooto.sigs.k8s.io/module-name$moduleName
oot.node.kubernetes.io/kernel-version.full$kernel
+

We either create those DaemonSets (if they do not already exist) or update them (if a DaemonSet already exists with +the same labels). +If a DaemonSet already exists for $moduleName and $kernel, it is updated with the new DriverContainer image. +If that image requires a build, we create the build object and wait for its completion before we modify the DaemonSet.

+

A new node joins the cluster (or is modified)

+

The OOTO lists all Modules and checks which ones should run on the node.
+For each Module that should run on the node (determined by the .spec.selector field), the operator tries to find +a suitable DriverContainer image for the node’s kernel.
+It creates a DaemonSet that runs that DriverContainer on all nodes running that kernel.

+

A node leaves the cluster

+

If that node was targeted by one of the DaemonSets, said DaemonSet’s .status.desiredNumberScheduled is +decremented.
+The operator watches changes to those objects; if that field reaches 0, the DaemonSet is deleted.

+

A node’s kernel is updated

+

Depending on the Kubernetes distribution used, this is functionally equivalent to either:

+
    +
  • Deleting the node and creating a new one, or;
    • +
  • Modifying an existing node.
    • +
+

That use case is thus covered above.

+

A node matches a Module’s .spec.selector, but not any of its kernel mappings

+

No DaemonSet is scheduled for this module and this kernel. The OOT kernel module will not be deployed on that node.

+
+
+ +
+
+

Documentation built with MkDocs.

+
+ + + + + + + diff --git a/docs/site/fonts/fontawesome-webfont.eot b/docs/site/fonts/fontawesome-webfont.eot new file mode 100644 index 000000000..e9f60ca95 Binary files /dev/null and b/docs/site/fonts/fontawesome-webfont.eot differ diff --git a/docs/site/fonts/fontawesome-webfont.svg b/docs/site/fonts/fontawesome-webfont.svg new file mode 100644 index 000000000..855c845e5 --- /dev/null +++ b/docs/site/fonts/fontawesome-webfont.svg @@ -0,0 +1,2671 @@ + + + + +Created by FontForge 20120731 at Mon Oct 24 17:37:40 2016 + By ,,, +Copyright Dave Gandy 2016. All rights reserved. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/site/fonts/fontawesome-webfont.ttf b/docs/site/fonts/fontawesome-webfont.ttf new file mode 100644 index 000000000..35acda2fa Binary files /dev/null and b/docs/site/fonts/fontawesome-webfont.ttf differ diff --git a/docs/site/fonts/fontawesome-webfont.woff b/docs/site/fonts/fontawesome-webfont.woff new file mode 100644 index 000000000..400014a4b Binary files /dev/null and b/docs/site/fonts/fontawesome-webfont.woff differ diff --git a/docs/site/fonts/fontawesome-webfont.woff2 b/docs/site/fonts/fontawesome-webfont.woff2 new file mode 100644 index 000000000..4d13fc604 Binary files /dev/null and b/docs/site/fonts/fontawesome-webfont.woff2 differ diff --git a/docs/site/img/favicon.ico b/docs/site/img/favicon.ico new file mode 100644 index 000000000..e85006a3c Binary files /dev/null and b/docs/site/img/favicon.ico differ diff --git a/docs/site/img/grid.png b/docs/site/img/grid.png new file mode 100644 index 000000000..878c3ed5c Binary files /dev/null and b/docs/site/img/grid.png differ diff --git a/docs/site/index.html b/docs/site/index.html new file mode 100644 index 000000000..43d720a59 --- /dev/null +++ b/docs/site/index.html @@ -0,0 +1,172 @@ + + + + + + + + + + + Out-Of-Tree operator + + + + + + + + + + + + +
+
+ Out-Of-Tree operator + + + + + +
+
+ +
+
+
+
+ +

Home

+

Kernel Module Management optionally builds and runs DriverContainers on Kubernetes clusters.

+

Getting started

+

Install the bleeding edge version of the KMM Operator by running the following command:

+
kubectl apply -k https://github.com/kubernetes-sigs/kernel-module-management/config/default
+
+
+
+ +
+
+

Documentation built with MkDocs.

+
+ + + + + + + + + diff --git a/docs/site/js/base.js b/docs/site/js/base.js new file mode 100644 index 000000000..b0f4726bf --- /dev/null +++ b/docs/site/js/base.js @@ -0,0 +1,283 @@ +function getSearchTerm() { + var sPageURL = window.location.search.substring(1); + var sURLVariables = sPageURL.split('&'); + for (var i = 0; i < sURLVariables.length; i++) { + var sParameterName = sURLVariables[i].split('='); + if (sParameterName[0] == 'q') { + return sParameterName[1]; + } + } +} + +function applyTopPadding() { + // Update various absolute positions to match where the main container + // starts. This is necessary for handling multi-line nav headers, since + // that pushes the main container down. + var offset = $('body > .container').offset(); + $('html').css('scroll-padding-top', offset.top + 'px'); + $('.bs-sidebar.affix').css('top', offset.top + 'px'); +} + +$(document).ready(function() { + + applyTopPadding(); + + var search_term = getSearchTerm(), + $search_modal = $('#mkdocs_search_modal'), + $keyboard_modal = $('#mkdocs_keyboard_modal'); + + if (search_term) { + $search_modal.modal(); + } + + // make sure search input gets autofocus every time modal opens. + $search_modal.on('shown.bs.modal', function() { + $search_modal.find('#mkdocs-search-query').focus(); + }); + + // Close search modal when result is selected + // The links get added later so listen to parent + $('#mkdocs-search-results').click(function(e) { + if ($(e.target).is('a')) { + $search_modal.modal('hide'); + } + }); + + // Populate keyboard modal with proper Keys + $keyboard_modal.find('.help.shortcut kbd')[0].innerHTML = keyCodes[shortcuts.help]; + $keyboard_modal.find('.prev.shortcut kbd')[0].innerHTML = keyCodes[shortcuts.previous]; + $keyboard_modal.find('.next.shortcut kbd')[0].innerHTML = keyCodes[shortcuts.next]; + $keyboard_modal.find('.search.shortcut kbd')[0].innerHTML = keyCodes[shortcuts.search]; + + // Keyboard navigation + document.addEventListener("keydown", function(e) { + if ($(e.target).is(':input')) return true; + var key = e.which || e.keyCode || window.event && window.event.keyCode; + var page; + switch (key) { + case shortcuts.next: + page = $('.navbar a[rel="next"]:first').prop('href'); + break; + case shortcuts.previous: + page = $('.navbar a[rel="prev"]:first').prop('href'); + break; + case shortcuts.search: + e.preventDefault(); + $keyboard_modal.modal('hide'); + $search_modal.modal('show'); + $search_modal.find('#mkdocs-search-query').focus(); + break; + case shortcuts.help: + $search_modal.modal('hide'); + $keyboard_modal.modal('show'); + break; + default: break; + } + if (page) { + $keyboard_modal.modal('hide'); + window.location.href = page; + } + }); + + $('table').addClass('table table-striped table-hover'); + + // Improve the scrollspy behaviour when users click on a TOC item. + $(".bs-sidenav a").on("click", function() { + var clicked = this; + setTimeout(function() { + var active = $('.nav li.active a'); + active = active[active.length - 1]; + if (clicked !== active) { + $(active).parent().removeClass("active"); + $(clicked).parent().addClass("active"); + } + }, 50); + }); + + function showInnerDropdown(item) { + var popup = $(item).next('.dropdown-menu'); + popup.addClass('show'); + $(item).addClass('open'); + + // First, close any sibling dropdowns. + var container = $(item).parent().parent(); + container.find('> .dropdown-submenu > a').each(function(i, el) { + if (el !== item) { + hideInnerDropdown(el); + } + }); + + var popupMargin = 10; + var maxBottom = $(window).height() - popupMargin; + var bounds = item.getBoundingClientRect(); + + popup.css('left', bounds.right + 'px'); + if (bounds.top + popup.height() > maxBottom && + bounds.top > $(window).height() / 2) { + popup.css({ + 'top': (bounds.bottom - popup.height()) + 'px', + 'max-height': (bounds.bottom - popupMargin) + 'px', + }); + } else { + popup.css({ + 'top': bounds.top + 'px', + 'max-height': (maxBottom - bounds.top) + 'px', + }); + } + } + + function hideInnerDropdown(item) { + var popup = $(item).next('.dropdown-menu'); + popup.removeClass('show'); + $(item).removeClass('open'); + + popup.scrollTop(0); + popup.find('.dropdown-menu').scrollTop(0).removeClass('show'); + popup.find('.dropdown-submenu > a').removeClass('open'); + } + + $('.dropdown-submenu > a').on('click', function(e) { + if ($(this).next('.dropdown-menu').hasClass('show')) { + hideInnerDropdown(this); + } else { + showInnerDropdown(this); + } + + e.stopPropagation(); + e.preventDefault(); + }); + + $('.dropdown-menu').parent().on('hide.bs.dropdown', function(e) { + $(this).find('.dropdown-menu').scrollTop(0); + $(this).find('.dropdown-submenu > a').removeClass('open'); + $(this).find('.dropdown-menu .dropdown-menu').removeClass('show'); + }); +}); + +$(window).on('resize', applyTopPadding); + +$('body').scrollspy({ + target: '.bs-sidebar', + offset: 100 +}); + +/* Prevent disabled links from causing a page reload */ +$("li.disabled a").click(function() { + event.preventDefault(); +}); + +// See https://www.cambiaresearch.com/articles/15/javascript-char-codes-key-codes +// We only list common keys below. Obscure keys are omitted and their use is discouraged. +var keyCodes = { + 8: 'backspace', + 9: 'tab', + 13: 'enter', + 16: 'shift', + 17: 'ctrl', + 18: 'alt', + 19: 'pause/break', + 20: 'caps lock', + 27: 'escape', + 32: 'spacebar', + 33: 'page up', + 34: 'page down', + 35: 'end', + 36: 'home', + 37: '←', + 38: '↑', + 39: '→', + 40: '↓', + 45: 'insert', + 46: 'delete', + 48: '0', + 49: '1', + 50: '2', + 51: '3', + 52: '4', + 53: '5', + 54: '6', + 55: '7', + 56: '8', + 57: '9', + 65: 'a', + 66: 'b', + 67: 'c', + 68: 'd', + 69: 'e', + 70: 'f', + 71: 'g', + 72: 'h', + 73: 'i', + 74: 'j', + 75: 'k', + 76: 'l', + 77: 'm', + 78: 'n', + 79: 'o', + 80: 'p', + 81: 'q', + 82: 'r', + 83: 's', + 84: 't', + 85: 'u', + 86: 'v', + 87: 'w', + 88: 'x', + 89: 'y', + 90: 'z', + 91: 'Left Windows Key / Left ⌘', + 92: 'Right Windows Key', + 93: 'Windows Menu / Right ⌘', + 96: 'numpad 0', + 97: 'numpad 1', + 98: 'numpad 2', + 99: 'numpad 3', + 100: 'numpad 4', + 101: 'numpad 5', + 102: 'numpad 6', + 103: 'numpad 7', + 104: 'numpad 8', + 105: 'numpad 9', + 106: 'multiply', + 107: 'add', + 109: 'subtract', + 110: 'decimal point', + 111: 'divide', + 112: 'f1', + 113: 'f2', + 114: 'f3', + 115: 'f4', + 116: 'f5', + 117: 'f6', + 118: 'f7', + 119: 'f8', + 120: 'f9', + 121: 'f10', + 122: 'f11', + 123: 'f12', + 124: 'f13', + 125: 'f14', + 126: 'f15', + 127: 'f16', + 128: 'f17', + 129: 'f18', + 130: 'f19', + 131: 'f20', + 132: 'f21', + 133: 'f22', + 134: 'f23', + 135: 'f24', + 144: 'num lock', + 145: 'scroll lock', + 186: ';', + 187: '=', + 188: ',', + 189: '‐', + 190: '.', + 191: '?', + 192: '`', + 219: '[', + 220: '\', + 221: ']', + 222: ''', +}; diff --git a/docs/site/js/bootstrap.min.js b/docs/site/js/bootstrap.min.js new file mode 100644 index 000000000..ca013b70f --- /dev/null +++ b/docs/site/js/bootstrap.min.js @@ -0,0 +1,7 @@ +/*! + * Bootstrap v4.3.1 (https://getbootstrap.com/) + * Copyright 2011-2019 The Bootstrap Authors (https://github.com/twbs/bootstrap/graphs/contributors) + * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) + */ +!function(t,e){"object"==typeof exports&&"undefined"!=typeof module?e(exports,require("jquery"),require("popper.js")):"function"==typeof define&&define.amd?define(["exports","jquery","popper.js"],e):e((t=t||self).bootstrap={},t.jQuery,t.Popper)}(this,function(t,g,u){"use strict";function i(t,e){for(var n=0;nthis._items.length-1||t<0))if(this._isSliding)g(this._element).one(Q.SLID,function(){return e.to(t)});else{if(n===t)return this.pause(),void this.cycle();var i=ndocument.documentElement.clientHeight;!this._isBodyOverflowing&&t&&(this._element.style.paddingLeft=this._scrollbarWidth+"px"),this._isBodyOverflowing&&!t&&(this._element.style.paddingRight=this._scrollbarWidth+"px")},t._resetAdjustments=function(){this._element.style.paddingLeft="",this._element.style.paddingRight=""},t._checkScrollbar=function(){var t=document.body.getBoundingClientRect();this._isBodyOverflowing=t.left+t.right
',trigger:"hover focus",title:"",delay:0,html:!1,selector:!1,placement:"top",offset:0,container:!1,fallbackPlacement:"flip",boundary:"scrollParent",sanitize:!0,sanitizeFn:null,whiteList:Ee},je="show",He="out",Re={HIDE:"hide"+De,HIDDEN:"hidden"+De,SHOW:"show"+De,SHOWN:"shown"+De,INSERTED:"inserted"+De,CLICK:"click"+De,FOCUSIN:"focusin"+De,FOCUSOUT:"focusout"+De,MOUSEENTER:"mouseenter"+De,MOUSELEAVE:"mouseleave"+De},xe="fade",Fe="show",Ue=".tooltip-inner",We=".arrow",qe="hover",Me="focus",Ke="click",Qe="manual",Be=function(){function i(t,e){if("undefined"==typeof u)throw new TypeError("Bootstrap's tooltips require Popper.js (https://popper.js.org/)");this._isEnabled=!0,this._timeout=0,this._hoverState="",this._activeTrigger={},this._popper=null,this.element=t,this.config=this._getConfig(e),this.tip=null,this._setListeners()}var t=i.prototype;return t.enable=function(){this._isEnabled=!0},t.disable=function(){this._isEnabled=!1},t.toggleEnabled=function(){this._isEnabled=!this._isEnabled},t.toggle=function(t){if(this._isEnabled)if(t){var e=this.constructor.DATA_KEY,n=g(t.currentTarget).data(e);n||(n=new this.constructor(t.currentTarget,this._getDelegateConfig()),g(t.currentTarget).data(e,n)),n._activeTrigger.click=!n._activeTrigger.click,n._isWithActiveTrigger()?n._enter(null,n):n._leave(null,n)}else{if(g(this.getTipElement()).hasClass(Fe))return void this._leave(null,this);this._enter(null,this)}},t.dispose=function(){clearTimeout(this._timeout),g.removeData(this.element,this.constructor.DATA_KEY),g(this.element).off(this.constructor.EVENT_KEY),g(this.element).closest(".modal").off("hide.bs.modal"),this.tip&&g(this.tip).remove(),this._isEnabled=null,this._timeout=null,this._hoverState=null,(this._activeTrigger=null)!==this._popper&&this._popper.destroy(),this._popper=null,this.element=null,this.config=null,this.tip=null},t.show=function(){var e=this;if("none"===g(this.element).css("display"))throw new Error("Please use show on visible elements");var t=g.Event(this.constructor.Event.SHOW);if(this.isWithContent()&&this._isEnabled){g(this.element).trigger(t);var n=_.findShadowRoot(this.element),i=g.contains(null!==n?n:this.element.ownerDocument.documentElement,this.element);if(t.isDefaultPrevented()||!i)return;var o=this.getTipElement(),r=_.getUID(this.constructor.NAME);o.setAttribute("id",r),this.element.setAttribute("aria-describedby",r),this.setContent(),this.config.animation&&g(o).addClass(xe);var s="function"==typeof this.config.placement?this.config.placement.call(this,o,this.element):this.config.placement,a=this._getAttachment(s);this.addAttachmentClass(a);var l=this._getContainer();g(o).data(this.constructor.DATA_KEY,this),g.contains(this.element.ownerDocument.documentElement,this.tip)||g(o).appendTo(l),g(this.element).trigger(this.constructor.Event.INSERTED),this._popper=new u(this.element,o,{placement:a,modifiers:{offset:this._getOffset(),flip:{behavior:this.config.fallbackPlacement},arrow:{element:We},preventOverflow:{boundariesElement:this.config.boundary}},onCreate:function(t){t.originalPlacement!==t.placement&&e._handlePopperPlacementChange(t)},onUpdate:function(t){return e._handlePopperPlacementChange(t)}}),g(o).addClass(Fe),"ontouchstart"in document.documentElement&&g(document.body).children().on("mouseover",null,g.noop);var c=function(){e.config.animation&&e._fixTransition();var t=e._hoverState;e._hoverState=null,g(e.element).trigger(e.constructor.Event.SHOWN),t===He&&e._leave(null,e)};if(g(this.tip).hasClass(xe)){var h=_.getTransitionDurationFromElement(this.tip);g(this.tip).one(_.TRANSITION_END,c).emulateTransitionEnd(h)}else c()}},t.hide=function(t){var e=this,n=this.getTipElement(),i=g.Event(this.constructor.Event.HIDE),o=function(){e._hoverState!==je&&n.parentNode&&n.parentNode.removeChild(n),e._cleanTipClass(),e.element.removeAttribute("aria-describedby"),g(e.element).trigger(e.constructor.Event.HIDDEN),null!==e._popper&&e._popper.destroy(),t&&t()};if(g(this.element).trigger(i),!i.isDefaultPrevented()){if(g(n).removeClass(Fe),"ontouchstart"in document.documentElement&&g(document.body).children().off("mouseover",null,g.noop),this._activeTrigger[Ke]=!1,this._activeTrigger[Me]=!1,this._activeTrigger[qe]=!1,g(this.tip).hasClass(xe)){var r=_.getTransitionDurationFromElement(n);g(n).one(_.TRANSITION_END,o).emulateTransitionEnd(r)}else o();this._hoverState=""}},t.update=function(){null!==this._popper&&this._popper.scheduleUpdate()},t.isWithContent=function(){return Boolean(this.getTitle())},t.addAttachmentClass=function(t){g(this.getTipElement()).addClass(Ae+"-"+t)},t.getTipElement=function(){return this.tip=this.tip||g(this.config.template)[0],this.tip},t.setContent=function(){var t=this.getTipElement();this.setElementContent(g(t.querySelectorAll(Ue)),this.getTitle()),g(t).removeClass(xe+" "+Fe)},t.setElementContent=function(t,e){"object"!=typeof e||!e.nodeType&&!e.jquery?this.config.html?(this.config.sanitize&&(e=Se(e,this.config.whiteList,this.config.sanitizeFn)),t.html(e)):t.text(e):this.config.html?g(e).parent().is(t)||t.empty().append(e):t.text(g(e).text())},t.getTitle=function(){var t=this.element.getAttribute("data-original-title");return t||(t="function"==typeof this.config.title?this.config.title.call(this.element):this.config.title),t},t._getOffset=function(){var e=this,t={};return"function"==typeof this.config.offset?t.fn=function(t){return t.offsets=l({},t.offsets,e.config.offset(t.offsets,e.element)||{}),t}:t.offset=this.config.offset,t},t._getContainer=function(){return!1===this.config.container?document.body:_.isElement(this.config.container)?g(this.config.container):g(document).find(this.config.container)},t._getAttachment=function(t){return Pe[t.toUpperCase()]},t._setListeners=function(){var i=this;this.config.trigger.split(" ").forEach(function(t){if("click"===t)g(i.element).on(i.constructor.Event.CLICK,i.config.selector,function(t){return i.toggle(t)});else if(t!==Qe){var e=t===qe?i.constructor.Event.MOUSEENTER:i.constructor.Event.FOCUSIN,n=t===qe?i.constructor.Event.MOUSELEAVE:i.constructor.Event.FOCUSOUT;g(i.element).on(e,i.config.selector,function(t){return i._enter(t)}).on(n,i.config.selector,function(t){return i._leave(t)})}}),g(this.element).closest(".modal").on("hide.bs.modal",function(){i.element&&i.hide()}),this.config.selector?this.config=l({},this.config,{trigger:"manual",selector:""}):this._fixTitle()},t._fixTitle=function(){var t=typeof this.element.getAttribute("data-original-title");(this.element.getAttribute("title")||"string"!==t)&&(this.element.setAttribute("data-original-title",this.element.getAttribute("title")||""),this.element.setAttribute("title",""))},t._enter=function(t,e){var n=this.constructor.DATA_KEY;(e=e||g(t.currentTarget).data(n))||(e=new this.constructor(t.currentTarget,this._getDelegateConfig()),g(t.currentTarget).data(n,e)),t&&(e._activeTrigger["focusin"===t.type?Me:qe]=!0),g(e.getTipElement()).hasClass(Fe)||e._hoverState===je?e._hoverState=je:(clearTimeout(e._timeout),e._hoverState=je,e.config.delay&&e.config.delay.show?e._timeout=setTimeout(function(){e._hoverState===je&&e.show()},e.config.delay.show):e.show())},t._leave=function(t,e){var n=this.constructor.DATA_KEY;(e=e||g(t.currentTarget).data(n))||(e=new this.constructor(t.currentTarget,this._getDelegateConfig()),g(t.currentTarget).data(n,e)),t&&(e._activeTrigger["focusout"===t.type?Me:qe]=!1),e._isWithActiveTrigger()||(clearTimeout(e._timeout),e._hoverState=He,e.config.delay&&e.config.delay.hide?e._timeout=setTimeout(function(){e._hoverState===He&&e.hide()},e.config.delay.hide):e.hide())},t._isWithActiveTrigger=function(){for(var t in this._activeTrigger)if(this._activeTrigger[t])return!0;return!1},t._getConfig=function(t){var e=g(this.element).data();return Object.keys(e).forEach(function(t){-1!==Oe.indexOf(t)&&delete e[t]}),"number"==typeof(t=l({},this.constructor.Default,e,"object"==typeof t&&t?t:{})).delay&&(t.delay={show:t.delay,hide:t.delay}),"number"==typeof t.title&&(t.title=t.title.toString()),"number"==typeof t.content&&(t.content=t.content.toString()),_.typeCheckConfig(be,t,this.constructor.DefaultType),t.sanitize&&(t.template=Se(t.template,t.whiteList,t.sanitizeFn)),t},t._getDelegateConfig=function(){var t={};if(this.config)for(var e in this.config)this.constructor.Default[e]!==this.config[e]&&(t[e]=this.config[e]);return t},t._cleanTipClass=function(){var t=g(this.getTipElement()),e=t.attr("class").match(Ne);null!==e&&e.length&&t.removeClass(e.join(""))},t._handlePopperPlacementChange=function(t){var e=t.instance;this.tip=e.popper,this._cleanTipClass(),this.addAttachmentClass(this._getAttachment(t.placement))},t._fixTransition=function(){var t=this.getTipElement(),e=this.config.animation;null===t.getAttribute("x-placement")&&(g(t).removeClass(xe),this.config.animation=!1,this.hide(),this.show(),this.config.animation=e)},i._jQueryInterface=function(n){return this.each(function(){var t=g(this).data(Ie),e="object"==typeof n&&n;if((t||!/dispose|hide/.test(n))&&(t||(t=new i(this,e),g(this).data(Ie,t)),"string"==typeof n)){if("undefined"==typeof t[n])throw new TypeError('No method named "'+n+'"');t[n]()}})},s(i,null,[{key:"VERSION",get:function(){return"4.3.1"}},{key:"Default",get:function(){return Le}},{key:"NAME",get:function(){return be}},{key:"DATA_KEY",get:function(){return Ie}},{key:"Event",get:function(){return Re}},{key:"EVENT_KEY",get:function(){return De}},{key:"DefaultType",get:function(){return ke}}]),i}();g.fn[be]=Be._jQueryInterface,g.fn[be].Constructor=Be,g.fn[be].noConflict=function(){return g.fn[be]=we,Be._jQueryInterface};var Ve="popover",Ye="bs.popover",ze="."+Ye,Xe=g.fn[Ve],$e="bs-popover",Ge=new RegExp("(^|\\s)"+$e+"\\S+","g"),Je=l({},Be.Default,{placement:"right",trigger:"click",content:"",template:''}),Ze=l({},Be.DefaultType,{content:"(string|element|function)"}),tn="fade",en="show",nn=".popover-header",on=".popover-body",rn={HIDE:"hide"+ze,HIDDEN:"hidden"+ze,SHOW:"show"+ze,SHOWN:"shown"+ze,INSERTED:"inserted"+ze,CLICK:"click"+ze,FOCUSIN:"focusin"+ze,FOCUSOUT:"focusout"+ze,MOUSEENTER:"mouseenter"+ze,MOUSELEAVE:"mouseleave"+ze},sn=function(t){var e,n;function i(){return t.apply(this,arguments)||this}n=t,(e=i).prototype=Object.create(n.prototype),(e.prototype.constructor=e).__proto__=n;var o=i.prototype;return o.isWithContent=function(){return this.getTitle()||this._getContent()},o.addAttachmentClass=function(t){g(this.getTipElement()).addClass($e+"-"+t)},o.getTipElement=function(){return this.tip=this.tip||g(this.config.template)[0],this.tip},o.setContent=function(){var t=g(this.getTipElement());this.setElementContent(t.find(nn),this.getTitle());var e=this._getContent();"function"==typeof e&&(e=e.call(this.element)),this.setElementContent(t.find(on),e),t.removeClass(tn+" "+en)},o._getContent=function(){return this.element.getAttribute("data-content")||this.config.content},o._cleanTipClass=function(){var t=g(this.getTipElement()),e=t.attr("class").match(Ge);null!==e&&0=this._offsets[o]&&("undefined"==typeof this._offsets[o+1]||t)[^>]*|#([\w-]*))$/,k=/^<(\w+)\s*\/?>(?:<\/\1>|)$/,E=/^[\],:{}\s]*$/,S=/(?:^|:|,)(?:\s*\[)+/g,A=/\\(?:["\\\/bfnrt]|u[\da-fA-F]{4})/g,j=/"[^"\\\r\n]*"|true|false|null|-?(?:\d+\.|)\d+(?:[eE][+-]?\d+|)/g,D=/^-ms-/,L=/-([\da-z])/gi,H=function(e,t){return t.toUpperCase()},q=function(e){(a.addEventListener||"load"===e.type||"complete"===a.readyState)&&(_(),x.ready())},_=function(){a.addEventListener?(a.removeEventListener("DOMContentLoaded",q,!1),e.removeEventListener("load",q,!1)):(a.detachEvent("onreadystatechange",q),e.detachEvent("onload",q))};x.fn=x.prototype={jquery:f,constructor:x,init:function(e,n,r){var i,o;if(!e)return this;if("string"==typeof e){if(i="<"===e.charAt(0)&&">"===e.charAt(e.length-1)&&e.length>=3?[null,e,null]:N.exec(e),!i||!i[1]&&n)return!n||n.jquery?(n||r).find(e):this.constructor(n).find(e);if(i[1]){if(n=n instanceof x?n[0]:n,x.merge(this,x.parseHTML(i[1],n&&n.nodeType?n.ownerDocument||n:a,!0)),k.test(i[1])&&x.isPlainObject(n))for(i in n)x.isFunction(this[i])?this[i](n[i]):this.attr(i,n[i]);return this}if(o=a.getElementById(i[2]),o&&o.parentNode){if(o.id!==i[2])return r.find(e);this.length=1,this[0]=o}return this.context=a,this.selector=e,this}return e.nodeType?(this.context=this[0]=e,this.length=1,this):x.isFunction(e)?r.ready(e):(e.selector!==t&&(this.selector=e.selector,this.context=e.context),x.makeArray(e,this))},selector:"",length:0,toArray:function(){return g.call(this)},get:function(e){return null==e?this.toArray():0>e?this[this.length+e]:this[e]},pushStack:function(e){var t=x.merge(this.constructor(),e);return t.prevObject=this,t.context=this.context,t},each:function(e,t){return x.each(this,e,t)},ready:function(e){return x.ready.promise().done(e),this},slice:function(){return this.pushStack(g.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},eq:function(e){var t=this.length,n=+e+(0>e?t:0);return this.pushStack(n>=0&&t>n?[this[n]]:[])},map:function(e){return this.pushStack(x.map(this,function(t,n){return e.call(t,n,t)}))},end:function(){return this.prevObject||this.constructor(null)},push:h,sort:[].sort,splice:[].splice},x.fn.init.prototype=x.fn,x.extend=x.fn.extend=function(){var e,n,r,i,o,a,s=arguments[0]||{},l=1,u=arguments.length,c=!1;for("boolean"==typeof s&&(c=s,s=arguments[1]||{},l=2),"object"==typeof s||x.isFunction(s)||(s={}),u===l&&(s=this,--l);u>l;l++)if(null!=(o=arguments[l]))for(i in o)e=s[i],r=o[i],s!==r&&(c&&r&&(x.isPlainObject(r)||(n=x.isArray(r)))?(n?(n=!1,a=e&&x.isArray(e)?e:[]):a=e&&x.isPlainObject(e)?e:{},s[i]=x.extend(c,a,r)):r!==t&&(s[i]=r));return s},x.extend({expando:"jQuery"+(f+Math.random()).replace(/\D/g,""),noConflict:function(t){return e.$===x&&(e.$=u),t&&e.jQuery===x&&(e.jQuery=l),x},isReady:!1,readyWait:1,holdReady:function(e){e?x.readyWait++:x.ready(!0)},ready:function(e){if(e===!0?!--x.readyWait:!x.isReady){if(!a.body)return setTimeout(x.ready);x.isReady=!0,e!==!0&&--x.readyWait>0||(n.resolveWith(a,[x]),x.fn.trigger&&x(a).trigger("ready").off("ready"))}},isFunction:function(e){return"function"===x.type(e)},isArray:Array.isArray||function(e){return"array"===x.type(e)},isWindow:function(e){return null!=e&&e==e.window},isNumeric:function(e){return!isNaN(parseFloat(e))&&isFinite(e)},type:function(e){return null==e?e+"":"object"==typeof e||"function"==typeof e?c[y.call(e)]||"object":typeof e},isPlainObject:function(e){var n;if(!e||"object"!==x.type(e)||e.nodeType||x.isWindow(e))return!1;try{if(e.constructor&&!v.call(e,"constructor")&&!v.call(e.constructor.prototype,"isPrototypeOf"))return!1}catch(r){return!1}if(x.support.ownLast)for(n in e)return v.call(e,n);for(n in e);return n===t||v.call(e,n)},isEmptyObject:function(e){var t;for(t in e)return!1;return!0},error:function(e){throw Error(e)},parseHTML:function(e,t,n){if(!e||"string"!=typeof e)return null;"boolean"==typeof t&&(n=t,t=!1),t=t||a;var r=k.exec(e),i=!n&&[];return r?[t.createElement(r[1])]:(r=x.buildFragment([e],t,i),i&&x(i).remove(),x.merge([],r.childNodes))},parseJSON:function(n){return e.JSON&&e.JSON.parse?e.JSON.parse(n):null===n?n:"string"==typeof n&&(n=x.trim(n),n&&E.test(n.replace(A,"@").replace(j,"]").replace(S,"")))?Function("return "+n)():(x.error("Invalid JSON: "+n),t)},parseXML:function(n){var r,i;if(!n||"string"!=typeof n)return null;try{e.DOMParser?(i=new DOMParser,r=i.parseFromString(n,"text/xml")):(r=new ActiveXObject("Microsoft.XMLDOM"),r.async="false",r.loadXML(n))}catch(o){r=t}return r&&r.documentElement&&!r.getElementsByTagName("parsererror").length||x.error("Invalid XML: "+n),r},noop:function(){},globalEval:function(t){t&&x.trim(t)&&(e.execScript||function(t){e.eval.call(e,t)})(t)},camelCase:function(e){return e.replace(D,"ms-").replace(L,H)},nodeName:function(e,t){return e.nodeName&&e.nodeName.toLowerCase()===t.toLowerCase()},each:function(e,t,n){var r,i=0,o=e.length,a=M(e);if(n){if(a){for(;o>i;i++)if(r=t.apply(e[i],n),r===!1)break}else for(i in e)if(r=t.apply(e[i],n),r===!1)break}else if(a){for(;o>i;i++)if(r=t.call(e[i],i,e[i]),r===!1)break}else for(i in e)if(r=t.call(e[i],i,e[i]),r===!1)break;return e},trim:b&&!b.call("\ufeff\u00a0")?function(e){return null==e?"":b.call(e)}:function(e){return null==e?"":(e+"").replace(C,"")},makeArray:function(e,t){var n=t||[];return null!=e&&(M(Object(e))?x.merge(n,"string"==typeof e?[e]:e):h.call(n,e)),n},inArray:function(e,t,n){var r;if(t){if(m)return m.call(t,e,n);for(r=t.length,n=n?0>n?Math.max(0,r+n):n:0;r>n;n++)if(n in t&&t[n]===e)return n}return-1},merge:function(e,n){var r=n.length,i=e.length,o=0;if("number"==typeof r)for(;r>o;o++)e[i++]=n[o];else while(n[o]!==t)e[i++]=n[o++];return e.length=i,e},grep:function(e,t,n){var r,i=[],o=0,a=e.length;for(n=!!n;a>o;o++)r=!!t(e[o],o),n!==r&&i.push(e[o]);return i},map:function(e,t,n){var r,i=0,o=e.length,a=M(e),s=[];if(a)for(;o>i;i++)r=t(e[i],i,n),null!=r&&(s[s.length]=r);else for(i in e)r=t(e[i],i,n),null!=r&&(s[s.length]=r);return d.apply([],s)},guid:1,proxy:function(e,n){var r,i,o;return"string"==typeof n&&(o=e[n],n=e,e=o),x.isFunction(e)?(r=g.call(arguments,2),i=function(){return e.apply(n||this,r.concat(g.call(arguments)))},i.guid=e.guid=e.guid||x.guid++,i):t},access:function(e,n,r,i,o,a,s){var l=0,u=e.length,c=null==r;if("object"===x.type(r)){o=!0;for(l in r)x.access(e,n,l,r[l],!0,a,s)}else if(i!==t&&(o=!0,x.isFunction(i)||(s=!0),c&&(s?(n.call(e,i),n=null):(c=n,n=function(e,t,n){return c.call(x(e),n)})),n))for(;u>l;l++)n(e[l],r,s?i:i.call(e[l],l,n(e[l],r)));return o?e:c?n.call(e):u?n(e[0],r):a},now:function(){return(new Date).getTime()},swap:function(e,t,n,r){var i,o,a={};for(o in t)a[o]=e.style[o],e.style[o]=t[o];i=n.apply(e,r||[]);for(o in t)e.style[o]=a[o];return i}}),x.ready.promise=function(t){if(!n)if(n=x.Deferred(),"complete"===a.readyState)setTimeout(x.ready);else if(a.addEventListener)a.addEventListener("DOMContentLoaded",q,!1),e.addEventListener("load",q,!1);else{a.attachEvent("onreadystatechange",q),e.attachEvent("onload",q);var r=!1;try{r=null==e.frameElement&&a.documentElement}catch(i){}r&&r.doScroll&&function o(){if(!x.isReady){try{r.doScroll("left")}catch(e){return setTimeout(o,50)}_(),x.ready()}}()}return n.promise(t)},x.each("Boolean Number String Function Array Date RegExp Object Error".split(" "),function(e,t){c["[object "+t+"]"]=t.toLowerCase()});function M(e){var t=e.length,n=x.type(e);return x.isWindow(e)?!1:1===e.nodeType&&t?!0:"array"===n||"function"!==n&&(0===t||"number"==typeof t&&t>0&&t-1 in e)}r=x(a),function(e,t){var n,r,i,o,a,s,l,u,c,p,f,d,h,g,m,y,v,b="sizzle"+-new Date,w=e.document,T=0,C=0,N=st(),k=st(),E=st(),S=!1,A=function(e,t){return e===t?(S=!0,0):0},j=typeof t,D=1<<31,L={}.hasOwnProperty,H=[],q=H.pop,_=H.push,M=H.push,O=H.slice,F=H.indexOf||function(e){var t=0,n=this.length;for(;n>t;t++)if(this[t]===e)return t;return-1},B="checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped",P="[\\x20\\t\\r\\n\\f]",R="(?:\\\\.|[\\w-]|[^\\x00-\\xa0])+",W=R.replace("w","w#"),$="\\["+P+"*("+R+")"+P+"*(?:([*^$|!~]?=)"+P+"*(?:(['\"])((?:\\\\.|[^\\\\])*?)\\3|("+W+")|)|)"+P+"*\\]",I=":("+R+")(?:\\(((['\"])((?:\\\\.|[^\\\\])*?)\\3|((?:\\\\.|[^\\\\()[\\]]|"+$.replace(3,8)+")*)|.*)\\)|)",z=RegExp("^"+P+"+|((?:^|[^\\\\])(?:\\\\.)*)"+P+"+$","g"),X=RegExp("^"+P+"*,"+P+"*"),U=RegExp("^"+P+"*([>+~]|"+P+")"+P+"*"),V=RegExp(P+"*[+~]"),Y=RegExp("="+P+"*([^\\]'\"]*)"+P+"*\\]","g"),J=RegExp(I),G=RegExp("^"+W+"$"),Q={ID:RegExp("^#("+R+")"),CLASS:RegExp("^\\.("+R+")"),TAG:RegExp("^("+R.replace("w","w*")+")"),ATTR:RegExp("^"+$),PSEUDO:RegExp("^"+I),CHILD:RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+P+"*(even|odd|(([+-]|)(\\d*)n|)"+P+"*(?:([+-]|)"+P+"*(\\d+)|))"+P+"*\\)|)","i"),bool:RegExp("^(?:"+B+")$","i"),needsContext:RegExp("^"+P+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+P+"*((?:-\\d)?\\d*)"+P+"*\\)|)(?=[^-]|$)","i")},K=/^[^{]+\{\s*\[native \w/,Z=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,et=/^(?:input|select|textarea|button)$/i,tt=/^h\d$/i,nt=/'|\\/g,rt=RegExp("\\\\([\\da-f]{1,6}"+P+"?|("+P+")|.)","ig"),it=function(e,t,n){var r="0x"+t-65536;return r!==r||n?t:0>r?String.fromCharCode(r+65536):String.fromCharCode(55296|r>>10,56320|1023&r)};try{M.apply(H=O.call(w.childNodes),w.childNodes),H[w.childNodes.length].nodeType}catch(ot){M={apply:H.length?function(e,t){_.apply(e,O.call(t))}:function(e,t){var n=e.length,r=0;while(e[n++]=t[r++]);e.length=n-1}}}function at(e,t,n,i){var o,a,s,l,u,c,d,m,y,x;if((t?t.ownerDocument||t:w)!==f&&p(t),t=t||f,n=n||[],!e||"string"!=typeof e)return n;if(1!==(l=t.nodeType)&&9!==l)return[];if(h&&!i){if(o=Z.exec(e))if(s=o[1]){if(9===l){if(a=t.getElementById(s),!a||!a.parentNode)return n;if(a.id===s)return n.push(a),n}else if(t.ownerDocument&&(a=t.ownerDocument.getElementById(s))&&v(t,a)&&a.id===s)return n.push(a),n}else{if(o[2])return M.apply(n,t.getElementsByTagName(e)),n;if((s=o[3])&&r.getElementsByClassName&&t.getElementsByClassName)return M.apply(n,t.getElementsByClassName(s)),n}if(r.qsa&&(!g||!g.test(e))){if(m=d=b,y=t,x=9===l&&e,1===l&&"object"!==t.nodeName.toLowerCase()){c=mt(e),(d=t.getAttribute("id"))?m=d.replace(nt,"\\$&"):t.setAttribute("id",m),m="[id='"+m+"'] ",u=c.length;while(u--)c[u]=m+yt(c[u]);y=V.test(e)&&t.parentNode||t,x=c.join(",")}if(x)try{return M.apply(n,y.querySelectorAll(x)),n}catch(T){}finally{d||t.removeAttribute("id")}}}return kt(e.replace(z,"$1"),t,n,i)}function st(){var e=[];function t(n,r){return e.push(n+=" ")>o.cacheLength&&delete t[e.shift()],t[n]=r}return t}function lt(e){return e[b]=!0,e}function ut(e){var t=f.createElement("div");try{return!!e(t)}catch(n){return!1}finally{t.parentNode&&t.parentNode.removeChild(t),t=null}}function ct(e,t){var n=e.split("|"),r=e.length;while(r--)o.attrHandle[n[r]]=t}function pt(e,t){var n=t&&e,r=n&&1===e.nodeType&&1===t.nodeType&&(~t.sourceIndex||D)-(~e.sourceIndex||D);if(r)return r;if(n)while(n=n.nextSibling)if(n===t)return-1;return e?1:-1}function ft(e){return function(t){var n=t.nodeName.toLowerCase();return"input"===n&&t.type===e}}function dt(e){return function(t){var n=t.nodeName.toLowerCase();return("input"===n||"button"===n)&&t.type===e}}function ht(e){return lt(function(t){return t=+t,lt(function(n,r){var i,o=e([],n.length,t),a=o.length;while(a--)n[i=o[a]]&&(n[i]=!(r[i]=n[i]))})})}s=at.isXML=function(e){var t=e&&(e.ownerDocument||e).documentElement;return t?"HTML"!==t.nodeName:!1},r=at.support={},p=at.setDocument=function(e){var n=e?e.ownerDocument||e:w,i=n.defaultView;return n!==f&&9===n.nodeType&&n.documentElement?(f=n,d=n.documentElement,h=!s(n),i&&i.attachEvent&&i!==i.top&&i.attachEvent("onbeforeunload",function(){p()}),r.attributes=ut(function(e){return e.className="i",!e.getAttribute("className")}),r.getElementsByTagName=ut(function(e){return e.appendChild(n.createComment("")),!e.getElementsByTagName("*").length}),r.getElementsByClassName=ut(function(e){return e.innerHTML="
",e.firstChild.className="i",2===e.getElementsByClassName("i").length}),r.getById=ut(function(e){return d.appendChild(e).id=b,!n.getElementsByName||!n.getElementsByName(b).length}),r.getById?(o.find.ID=function(e,t){if(typeof t.getElementById!==j&&h){var n=t.getElementById(e);return n&&n.parentNode?[n]:[]}},o.filter.ID=function(e){var t=e.replace(rt,it);return function(e){return e.getAttribute("id")===t}}):(delete o.find.ID,o.filter.ID=function(e){var t=e.replace(rt,it);return function(e){var n=typeof e.getAttributeNode!==j&&e.getAttributeNode("id");return n&&n.value===t}}),o.find.TAG=r.getElementsByTagName?function(e,n){return typeof n.getElementsByTagName!==j?n.getElementsByTagName(e):t}:function(e,t){var n,r=[],i=0,o=t.getElementsByTagName(e);if("*"===e){while(n=o[i++])1===n.nodeType&&r.push(n);return r}return o},o.find.CLASS=r.getElementsByClassName&&function(e,n){return typeof n.getElementsByClassName!==j&&h?n.getElementsByClassName(e):t},m=[],g=[],(r.qsa=K.test(n.querySelectorAll))&&(ut(function(e){e.innerHTML="",e.querySelectorAll("[selected]").length||g.push("\\["+P+"*(?:value|"+B+")"),e.querySelectorAll(":checked").length||g.push(":checked")}),ut(function(e){var t=n.createElement("input");t.setAttribute("type","hidden"),e.appendChild(t).setAttribute("t",""),e.querySelectorAll("[t^='']").length&&g.push("[*^$]="+P+"*(?:''|\"\")"),e.querySelectorAll(":enabled").length||g.push(":enabled",":disabled"),e.querySelectorAll("*,:x"),g.push(",.*:")})),(r.matchesSelector=K.test(y=d.webkitMatchesSelector||d.mozMatchesSelector||d.oMatchesSelector||d.msMatchesSelector))&&ut(function(e){r.disconnectedMatch=y.call(e,"div"),y.call(e,"[s!='']:x"),m.push("!=",I)}),g=g.length&&RegExp(g.join("|")),m=m.length&&RegExp(m.join("|")),v=K.test(d.contains)||d.compareDocumentPosition?function(e,t){var n=9===e.nodeType?e.documentElement:e,r=t&&t.parentNode;return e===r||!(!r||1!==r.nodeType||!(n.contains?n.contains(r):e.compareDocumentPosition&&16&e.compareDocumentPosition(r)))}:function(e,t){if(t)while(t=t.parentNode)if(t===e)return!0;return!1},A=d.compareDocumentPosition?function(e,t){if(e===t)return S=!0,0;var i=t.compareDocumentPosition&&e.compareDocumentPosition&&e.compareDocumentPosition(t);return i?1&i||!r.sortDetached&&t.compareDocumentPosition(e)===i?e===n||v(w,e)?-1:t===n||v(w,t)?1:c?F.call(c,e)-F.call(c,t):0:4&i?-1:1:e.compareDocumentPosition?-1:1}:function(e,t){var r,i=0,o=e.parentNode,a=t.parentNode,s=[e],l=[t];if(e===t)return S=!0,0;if(!o||!a)return e===n?-1:t===n?1:o?-1:a?1:c?F.call(c,e)-F.call(c,t):0;if(o===a)return pt(e,t);r=e;while(r=r.parentNode)s.unshift(r);r=t;while(r=r.parentNode)l.unshift(r);while(s[i]===l[i])i++;return i?pt(s[i],l[i]):s[i]===w?-1:l[i]===w?1:0},n):f},at.matches=function(e,t){return at(e,null,null,t)},at.matchesSelector=function(e,t){if((e.ownerDocument||e)!==f&&p(e),t=t.replace(Y,"='$1']"),!(!r.matchesSelector||!h||m&&m.test(t)||g&&g.test(t)))try{var n=y.call(e,t);if(n||r.disconnectedMatch||e.document&&11!==e.document.nodeType)return n}catch(i){}return at(t,f,null,[e]).length>0},at.contains=function(e,t){return(e.ownerDocument||e)!==f&&p(e),v(e,t)},at.attr=function(e,n){(e.ownerDocument||e)!==f&&p(e);var i=o.attrHandle[n.toLowerCase()],a=i&&L.call(o.attrHandle,n.toLowerCase())?i(e,n,!h):t;return a===t?r.attributes||!h?e.getAttribute(n):(a=e.getAttributeNode(n))&&a.specified?a.value:null:a},at.error=function(e){throw Error("Syntax error, unrecognized expression: "+e)},at.uniqueSort=function(e){var t,n=[],i=0,o=0;if(S=!r.detectDuplicates,c=!r.sortStable&&e.slice(0),e.sort(A),S){while(t=e[o++])t===e[o]&&(i=n.push(o));while(i--)e.splice(n[i],1)}return e},a=at.getText=function(e){var t,n="",r=0,i=e.nodeType;if(i){if(1===i||9===i||11===i){if("string"==typeof e.textContent)return e.textContent;for(e=e.firstChild;e;e=e.nextSibling)n+=a(e)}else if(3===i||4===i)return e.nodeValue}else for(;t=e[r];r++)n+=a(t);return n},o=at.selectors={cacheLength:50,createPseudo:lt,match:Q,attrHandle:{},find:{},relative:{">":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(e){return e[1]=e[1].replace(rt,it),e[3]=(e[4]||e[5]||"").replace(rt,it),"~="===e[2]&&(e[3]=" "+e[3]+" "),e.slice(0,4)},CHILD:function(e){return e[1]=e[1].toLowerCase(),"nth"===e[1].slice(0,3)?(e[3]||at.error(e[0]),e[4]=+(e[4]?e[5]+(e[6]||1):2*("even"===e[3]||"odd"===e[3])),e[5]=+(e[7]+e[8]||"odd"===e[3])):e[3]&&at.error(e[0]),e},PSEUDO:function(e){var n,r=!e[5]&&e[2];return Q.CHILD.test(e[0])?null:(e[3]&&e[4]!==t?e[2]=e[4]:r&&J.test(r)&&(n=mt(r,!0))&&(n=r.indexOf(")",r.length-n)-r.length)&&(e[0]=e[0].slice(0,n),e[2]=r.slice(0,n)),e.slice(0,3))}},filter:{TAG:function(e){var t=e.replace(rt,it).toLowerCase();return"*"===e?function(){return!0}:function(e){return e.nodeName&&e.nodeName.toLowerCase()===t}},CLASS:function(e){var t=N[e+" "];return t||(t=RegExp("(^|"+P+")"+e+"("+P+"|$)"))&&N(e,function(e){return t.test("string"==typeof e.className&&e.className||typeof e.getAttribute!==j&&e.getAttribute("class")||"")})},ATTR:function(e,t,n){return function(r){var i=at.attr(r,e);return null==i?"!="===t:t?(i+="","="===t?i===n:"!="===t?i!==n:"^="===t?n&&0===i.indexOf(n):"*="===t?n&&i.indexOf(n)>-1:"$="===t?n&&i.slice(-n.length)===n:"~="===t?(" "+i+" ").indexOf(n)>-1:"|="===t?i===n||i.slice(0,n.length+1)===n+"-":!1):!0}},CHILD:function(e,t,n,r,i){var o="nth"!==e.slice(0,3),a="last"!==e.slice(-4),s="of-type"===t;return 1===r&&0===i?function(e){return!!e.parentNode}:function(t,n,l){var u,c,p,f,d,h,g=o!==a?"nextSibling":"previousSibling",m=t.parentNode,y=s&&t.nodeName.toLowerCase(),v=!l&&!s;if(m){if(o){while(g){p=t;while(p=p[g])if(s?p.nodeName.toLowerCase()===y:1===p.nodeType)return!1;h=g="only"===e&&!h&&"nextSibling"}return!0}if(h=[a?m.firstChild:m.lastChild],a&&v){c=m[b]||(m[b]={}),u=c[e]||[],d=u[0]===T&&u[1],f=u[0]===T&&u[2],p=d&&m.childNodes[d];while(p=++d&&p&&p[g]||(f=d=0)||h.pop())if(1===p.nodeType&&++f&&p===t){c[e]=[T,d,f];break}}else if(v&&(u=(t[b]||(t[b]={}))[e])&&u[0]===T)f=u[1];else while(p=++d&&p&&p[g]||(f=d=0)||h.pop())if((s?p.nodeName.toLowerCase()===y:1===p.nodeType)&&++f&&(v&&((p[b]||(p[b]={}))[e]=[T,f]),p===t))break;return f-=i,f===r||0===f%r&&f/r>=0}}},PSEUDO:function(e,t){var n,r=o.pseudos[e]||o.setFilters[e.toLowerCase()]||at.error("unsupported pseudo: "+e);return r[b]?r(t):r.length>1?(n=[e,e,"",t],o.setFilters.hasOwnProperty(e.toLowerCase())?lt(function(e,n){var i,o=r(e,t),a=o.length;while(a--)i=F.call(e,o[a]),e[i]=!(n[i]=o[a])}):function(e){return r(e,0,n)}):r}},pseudos:{not:lt(function(e){var t=[],n=[],r=l(e.replace(z,"$1"));return r[b]?lt(function(e,t,n,i){var o,a=r(e,null,i,[]),s=e.length;while(s--)(o=a[s])&&(e[s]=!(t[s]=o))}):function(e,i,o){return t[0]=e,r(t,null,o,n),!n.pop()}}),has:lt(function(e){return function(t){return at(e,t).length>0}}),contains:lt(function(e){return function(t){return(t.textContent||t.innerText||a(t)).indexOf(e)>-1}}),lang:lt(function(e){return G.test(e||"")||at.error("unsupported lang: "+e),e=e.replace(rt,it).toLowerCase(),function(t){var n;do if(n=h?t.lang:t.getAttribute("xml:lang")||t.getAttribute("lang"))return n=n.toLowerCase(),n===e||0===n.indexOf(e+"-");while((t=t.parentNode)&&1===t.nodeType);return!1}}),target:function(t){var n=e.location&&e.location.hash;return n&&n.slice(1)===t.id},root:function(e){return e===d},focus:function(e){return e===f.activeElement&&(!f.hasFocus||f.hasFocus())&&!!(e.type||e.href||~e.tabIndex)},enabled:function(e){return e.disabled===!1},disabled:function(e){return e.disabled===!0},checked:function(e){var t=e.nodeName.toLowerCase();return"input"===t&&!!e.checked||"option"===t&&!!e.selected},selected:function(e){return e.parentNode&&e.parentNode.selectedIndex,e.selected===!0},empty:function(e){for(e=e.firstChild;e;e=e.nextSibling)if(e.nodeName>"@"||3===e.nodeType||4===e.nodeType)return!1;return!0},parent:function(e){return!o.pseudos.empty(e)},header:function(e){return tt.test(e.nodeName)},input:function(e){return et.test(e.nodeName)},button:function(e){var t=e.nodeName.toLowerCase();return"input"===t&&"button"===e.type||"button"===t},text:function(e){var t;return"input"===e.nodeName.toLowerCase()&&"text"===e.type&&(null==(t=e.getAttribute("type"))||t.toLowerCase()===e.type)},first:ht(function(){return[0]}),last:ht(function(e,t){return[t-1]}),eq:ht(function(e,t,n){return[0>n?n+t:n]}),even:ht(function(e,t){var n=0;for(;t>n;n+=2)e.push(n);return e}),odd:ht(function(e,t){var n=1;for(;t>n;n+=2)e.push(n);return e}),lt:ht(function(e,t,n){var r=0>n?n+t:n;for(;--r>=0;)e.push(r);return e}),gt:ht(function(e,t,n){var r=0>n?n+t:n;for(;t>++r;)e.push(r);return e})}},o.pseudos.nth=o.pseudos.eq;for(n in{radio:!0,checkbox:!0,file:!0,password:!0,image:!0})o.pseudos[n]=ft(n);for(n in{submit:!0,reset:!0})o.pseudos[n]=dt(n);function gt(){}gt.prototype=o.filters=o.pseudos,o.setFilters=new gt;function mt(e,t){var n,r,i,a,s,l,u,c=k[e+" "];if(c)return t?0:c.slice(0);s=e,l=[],u=o.preFilter;while(s){(!n||(r=X.exec(s)))&&(r&&(s=s.slice(r[0].length)||s),l.push(i=[])),n=!1,(r=U.exec(s))&&(n=r.shift(),i.push({value:n,type:r[0].replace(z," ")}),s=s.slice(n.length));for(a in o.filter)!(r=Q[a].exec(s))||u[a]&&!(r=u[a](r))||(n=r.shift(),i.push({value:n,type:a,matches:r}),s=s.slice(n.length));if(!n)break}return t?s.length:s?at.error(e):k(e,l).slice(0)}function yt(e){var t=0,n=e.length,r="";for(;n>t;t++)r+=e[t].value;return r}function vt(e,t,n){var r=t.dir,o=n&&"parentNode"===r,a=C++;return t.first?function(t,n,i){while(t=t[r])if(1===t.nodeType||o)return e(t,n,i)}:function(t,n,s){var l,u,c,p=T+" "+a;if(s){while(t=t[r])if((1===t.nodeType||o)&&e(t,n,s))return!0}else while(t=t[r])if(1===t.nodeType||o)if(c=t[b]||(t[b]={}),(u=c[r])&&u[0]===p){if((l=u[1])===!0||l===i)return l===!0}else if(u=c[r]=[p],u[1]=e(t,n,s)||i,u[1]===!0)return!0}}function bt(e){return e.length>1?function(t,n,r){var i=e.length;while(i--)if(!e[i](t,n,r))return!1;return!0}:e[0]}function xt(e,t,n,r,i){var o,a=[],s=0,l=e.length,u=null!=t;for(;l>s;s++)(o=e[s])&&(!n||n(o,r,i))&&(a.push(o),u&&t.push(s));return a}function wt(e,t,n,r,i,o){return r&&!r[b]&&(r=wt(r)),i&&!i[b]&&(i=wt(i,o)),lt(function(o,a,s,l){var u,c,p,f=[],d=[],h=a.length,g=o||Nt(t||"*",s.nodeType?[s]:s,[]),m=!e||!o&&t?g:xt(g,f,e,s,l),y=n?i||(o?e:h||r)?[]:a:m;if(n&&n(m,y,s,l),r){u=xt(y,d),r(u,[],s,l),c=u.length;while(c--)(p=u[c])&&(y[d[c]]=!(m[d[c]]=p))}if(o){if(i||e){if(i){u=[],c=y.length;while(c--)(p=y[c])&&u.push(m[c]=p);i(null,y=[],u,l)}c=y.length;while(c--)(p=y[c])&&(u=i?F.call(o,p):f[c])>-1&&(o[u]=!(a[u]=p))}}else y=xt(y===a?y.splice(h,y.length):y),i?i(null,a,y,l):M.apply(a,y)})}function Tt(e){var t,n,r,i=e.length,a=o.relative[e[0].type],s=a||o.relative[" "],l=a?1:0,c=vt(function(e){return e===t},s,!0),p=vt(function(e){return F.call(t,e)>-1},s,!0),f=[function(e,n,r){return!a&&(r||n!==u)||((t=n).nodeType?c(e,n,r):p(e,n,r))}];for(;i>l;l++)if(n=o.relative[e[l].type])f=[vt(bt(f),n)];else{if(n=o.filter[e[l].type].apply(null,e[l].matches),n[b]){for(r=++l;i>r;r++)if(o.relative[e[r].type])break;return wt(l>1&&bt(f),l>1&&yt(e.slice(0,l-1).concat({value:" "===e[l-2].type?"*":""})).replace(z,"$1"),n,r>l&&Tt(e.slice(l,r)),i>r&&Tt(e=e.slice(r)),i>r&&yt(e))}f.push(n)}return bt(f)}function Ct(e,t){var n=0,r=t.length>0,a=e.length>0,s=function(s,l,c,p,d){var h,g,m,y=[],v=0,b="0",x=s&&[],w=null!=d,C=u,N=s||a&&o.find.TAG("*",d&&l.parentNode||l),k=T+=null==C?1:Math.random()||.1;for(w&&(u=l!==f&&l,i=n);null!=(h=N[b]);b++){if(a&&h){g=0;while(m=e[g++])if(m(h,l,c)){p.push(h);break}w&&(T=k,i=++n)}r&&((h=!m&&h)&&v--,s&&x.push(h))}if(v+=b,r&&b!==v){g=0;while(m=t[g++])m(x,y,l,c);if(s){if(v>0)while(b--)x[b]||y[b]||(y[b]=q.call(p));y=xt(y)}M.apply(p,y),w&&!s&&y.length>0&&v+t.length>1&&at.uniqueSort(p)}return w&&(T=k,u=C),x};return r?lt(s):s}l=at.compile=function(e,t){var n,r=[],i=[],o=E[e+" "];if(!o){t||(t=mt(e)),n=t.length;while(n--)o=Tt(t[n]),o[b]?r.push(o):i.push(o);o=E(e,Ct(i,r))}return o};function Nt(e,t,n){var r=0,i=t.length;for(;i>r;r++)at(e,t[r],n);return n}function kt(e,t,n,i){var a,s,u,c,p,f=mt(e);if(!i&&1===f.length){if(s=f[0]=f[0].slice(0),s.length>2&&"ID"===(u=s[0]).type&&r.getById&&9===t.nodeType&&h&&o.relative[s[1].type]){if(t=(o.find.ID(u.matches[0].replace(rt,it),t)||[])[0],!t)return n;e=e.slice(s.shift().value.length)}a=Q.needsContext.test(e)?0:s.length;while(a--){if(u=s[a],o.relative[c=u.type])break;if((p=o.find[c])&&(i=p(u.matches[0].replace(rt,it),V.test(s[0].type)&&t.parentNode||t))){if(s.splice(a,1),e=i.length&&yt(s),!e)return M.apply(n,i),n;break}}}return l(e,f)(i,t,!h,n,V.test(e)),n}r.sortStable=b.split("").sort(A).join("")===b,r.detectDuplicates=S,p(),r.sortDetached=ut(function(e){return 1&e.compareDocumentPosition(f.createElement("div"))}),ut(function(e){return e.innerHTML="","#"===e.firstChild.getAttribute("href")})||ct("type|href|height|width",function(e,n,r){return r?t:e.getAttribute(n,"type"===n.toLowerCase()?1:2)}),r.attributes&&ut(function(e){return e.innerHTML="",e.firstChild.setAttribute("value",""),""===e.firstChild.getAttribute("value")})||ct("value",function(e,n,r){return r||"input"!==e.nodeName.toLowerCase()?t:e.defaultValue}),ut(function(e){return null==e.getAttribute("disabled")})||ct(B,function(e,n,r){var i;return r?t:(i=e.getAttributeNode(n))&&i.specified?i.value:e[n]===!0?n.toLowerCase():null}),x.find=at,x.expr=at.selectors,x.expr[":"]=x.expr.pseudos,x.unique=at.uniqueSort,x.text=at.getText,x.isXMLDoc=at.isXML,x.contains=at.contains}(e);var O={};function F(e){var t=O[e]={};return x.each(e.match(T)||[],function(e,n){t[n]=!0}),t}x.Callbacks=function(e){e="string"==typeof e?O[e]||F(e):x.extend({},e);var n,r,i,o,a,s,l=[],u=!e.once&&[],c=function(t){for(r=e.memory&&t,i=!0,a=s||0,s=0,o=l.length,n=!0;l&&o>a;a++)if(l[a].apply(t[0],t[1])===!1&&e.stopOnFalse){r=!1;break}n=!1,l&&(u?u.length&&c(u.shift()):r?l=[]:p.disable())},p={add:function(){if(l){var t=l.length;(function i(t){x.each(t,function(t,n){var r=x.type(n);"function"===r?e.unique&&p.has(n)||l.push(n):n&&n.length&&"string"!==r&&i(n)})})(arguments),n?o=l.length:r&&(s=t,c(r))}return this},remove:function(){return l&&x.each(arguments,function(e,t){var r;while((r=x.inArray(t,l,r))>-1)l.splice(r,1),n&&(o>=r&&o--,a>=r&&a--)}),this},has:function(e){return e?x.inArray(e,l)>-1:!(!l||!l.length)},empty:function(){return l=[],o=0,this},disable:function(){return l=u=r=t,this},disabled:function(){return!l},lock:function(){return u=t,r||p.disable(),this},locked:function(){return!u},fireWith:function(e,t){return!l||i&&!u||(t=t||[],t=[e,t.slice?t.slice():t],n?u.push(t):c(t)),this},fire:function(){return p.fireWith(this,arguments),this},fired:function(){return!!i}};return p},x.extend({Deferred:function(e){var t=[["resolve","done",x.Callbacks("once memory"),"resolved"],["reject","fail",x.Callbacks("once memory"),"rejected"],["notify","progress",x.Callbacks("memory")]],n="pending",r={state:function(){return n},always:function(){return i.done(arguments).fail(arguments),this},then:function(){var e=arguments;return x.Deferred(function(n){x.each(t,function(t,o){var a=o[0],s=x.isFunction(e[t])&&e[t];i[o[1]](function(){var e=s&&s.apply(this,arguments);e&&x.isFunction(e.promise)?e.promise().done(n.resolve).fail(n.reject).progress(n.notify):n[a+"With"](this===r?n.promise():this,s?[e]:arguments)})}),e=null}).promise()},promise:function(e){return null!=e?x.extend(e,r):r}},i={};return r.pipe=r.then,x.each(t,function(e,o){var a=o[2],s=o[3];r[o[1]]=a.add,s&&a.add(function(){n=s},t[1^e][2].disable,t[2][2].lock),i[o[0]]=function(){return i[o[0]+"With"](this===i?r:this,arguments),this},i[o[0]+"With"]=a.fireWith}),r.promise(i),e&&e.call(i,i),i},when:function(e){var t=0,n=g.call(arguments),r=n.length,i=1!==r||e&&x.isFunction(e.promise)?r:0,o=1===i?e:x.Deferred(),a=function(e,t,n){return function(r){t[e]=this,n[e]=arguments.length>1?g.call(arguments):r,n===s?o.notifyWith(t,n):--i||o.resolveWith(t,n)}},s,l,u;if(r>1)for(s=Array(r),l=Array(r),u=Array(r);r>t;t++)n[t]&&x.isFunction(n[t].promise)?n[t].promise().done(a(t,u,n)).fail(o.reject).progress(a(t,l,s)):--i;return i||o.resolveWith(u,n),o.promise()}}),x.support=function(t){var n,r,o,s,l,u,c,p,f,d=a.createElement("div");if(d.setAttribute("className","t"),d.innerHTML="
a",n=d.getElementsByTagName("*")||[],r=d.getElementsByTagName("a")[0],!r||!r.style||!n.length)return t;s=a.createElement("select"),u=s.appendChild(a.createElement("option")),o=d.getElementsByTagName("input")[0],r.style.cssText="top:1px;float:left;opacity:.5",t.getSetAttribute="t"!==d.className,t.leadingWhitespace=3===d.firstChild.nodeType,t.tbody=!d.getElementsByTagName("tbody").length,t.htmlSerialize=!!d.getElementsByTagName("link").length,t.style=/top/.test(r.getAttribute("style")),t.hrefNormalized="/a"===r.getAttribute("href"),t.opacity=/^0.5/.test(r.style.opacity),t.cssFloat=!!r.style.cssFloat,t.checkOn=!!o.value,t.optSelected=u.selected,t.enctype=!!a.createElement("form").enctype,t.html5Clone="<:nav>"!==a.createElement("nav").cloneNode(!0).outerHTML,t.inlineBlockNeedsLayout=!1,t.shrinkWrapBlocks=!1,t.pixelPosition=!1,t.deleteExpando=!0,t.noCloneEvent=!0,t.reliableMarginRight=!0,t.boxSizingReliable=!0,o.checked=!0,t.noCloneChecked=o.cloneNode(!0).checked,s.disabled=!0,t.optDisabled=!u.disabled;try{delete d.test}catch(h){t.deleteExpando=!1}o=a.createElement("input"),o.setAttribute("value",""),t.input=""===o.getAttribute("value"),o.value="t",o.setAttribute("type","radio"),t.radioValue="t"===o.value,o.setAttribute("checked","t"),o.setAttribute("name","t"),l=a.createDocumentFragment(),l.appendChild(o),t.appendChecked=o.checked,t.checkClone=l.cloneNode(!0).cloneNode(!0).lastChild.checked,d.attachEvent&&(d.attachEvent("onclick",function(){t.noCloneEvent=!1}),d.cloneNode(!0).click());for(f in{submit:!0,change:!0,focusin:!0})d.setAttribute(c="on"+f,"t"),t[f+"Bubbles"]=c in e||d.attributes[c].expando===!1;d.style.backgroundClip="content-box",d.cloneNode(!0).style.backgroundClip="",t.clearCloneStyle="content-box"===d.style.backgroundClip;for(f in x(t))break;return t.ownLast="0"!==f,x(function(){var n,r,o,s="padding:0;margin:0;border:0;display:block;box-sizing:content-box;-moz-box-sizing:content-box;-webkit-box-sizing:content-box;",l=a.getElementsByTagName("body")[0];l&&(n=a.createElement("div"),n.style.cssText="border:0;width:0;height:0;position:absolute;top:0;left:-9999px;margin-top:1px",l.appendChild(n).appendChild(d),d.innerHTML="
t
",o=d.getElementsByTagName("td"),o[0].style.cssText="padding:0;margin:0;border:0;display:none",p=0===o[0].offsetHeight,o[0].style.display="",o[1].style.display="none",t.reliableHiddenOffsets=p&&0===o[0].offsetHeight,d.innerHTML="",d.style.cssText="box-sizing:border-box;-moz-box-sizing:border-box;-webkit-box-sizing:border-box;padding:1px;border:1px;display:block;width:4px;margin-top:1%;position:absolute;top:1%;",x.swap(l,null!=l.style.zoom?{zoom:1}:{},function(){t.boxSizing=4===d.offsetWidth}),e.getComputedStyle&&(t.pixelPosition="1%"!==(e.getComputedStyle(d,null)||{}).top,t.boxSizingReliable="4px"===(e.getComputedStyle(d,null)||{width:"4px"}).width,r=d.appendChild(a.createElement("div")),r.style.cssText=d.style.cssText=s,r.style.marginRight=r.style.width="0",d.style.width="1px",t.reliableMarginRight=!parseFloat((e.getComputedStyle(r,null)||{}).marginRight)),typeof d.style.zoom!==i&&(d.innerHTML="",d.style.cssText=s+"width:1px;padding:1px;display:inline;zoom:1",t.inlineBlockNeedsLayout=3===d.offsetWidth,d.style.display="block",d.innerHTML="
",d.firstChild.style.width="5px",t.shrinkWrapBlocks=3!==d.offsetWidth,t.inlineBlockNeedsLayout&&(l.style.zoom=1)),l.removeChild(n),n=d=o=r=null)}),n=s=l=u=r=o=null,t +}({});var B=/(?:\{[\s\S]*\}|\[[\s\S]*\])$/,P=/([A-Z])/g;function R(e,n,r,i){if(x.acceptData(e)){var o,a,s=x.expando,l=e.nodeType,u=l?x.cache:e,c=l?e[s]:e[s]&&s;if(c&&u[c]&&(i||u[c].data)||r!==t||"string"!=typeof n)return c||(c=l?e[s]=p.pop()||x.guid++:s),u[c]||(u[c]=l?{}:{toJSON:x.noop}),("object"==typeof n||"function"==typeof n)&&(i?u[c]=x.extend(u[c],n):u[c].data=x.extend(u[c].data,n)),a=u[c],i||(a.data||(a.data={}),a=a.data),r!==t&&(a[x.camelCase(n)]=r),"string"==typeof n?(o=a[n],null==o&&(o=a[x.camelCase(n)])):o=a,o}}function W(e,t,n){if(x.acceptData(e)){var r,i,o=e.nodeType,a=o?x.cache:e,s=o?e[x.expando]:x.expando;if(a[s]){if(t&&(r=n?a[s]:a[s].data)){x.isArray(t)?t=t.concat(x.map(t,x.camelCase)):t in r?t=[t]:(t=x.camelCase(t),t=t in r?[t]:t.split(" ")),i=t.length;while(i--)delete r[t[i]];if(n?!I(r):!x.isEmptyObject(r))return}(n||(delete a[s].data,I(a[s])))&&(o?x.cleanData([e],!0):x.support.deleteExpando||a!=a.window?delete a[s]:a[s]=null)}}}x.extend({cache:{},noData:{applet:!0,embed:!0,object:"clsid:D27CDB6E-AE6D-11cf-96B8-444553540000"},hasData:function(e){return e=e.nodeType?x.cache[e[x.expando]]:e[x.expando],!!e&&!I(e)},data:function(e,t,n){return R(e,t,n)},removeData:function(e,t){return W(e,t)},_data:function(e,t,n){return R(e,t,n,!0)},_removeData:function(e,t){return W(e,t,!0)},acceptData:function(e){if(e.nodeType&&1!==e.nodeType&&9!==e.nodeType)return!1;var t=e.nodeName&&x.noData[e.nodeName.toLowerCase()];return!t||t!==!0&&e.getAttribute("classid")===t}}),x.fn.extend({data:function(e,n){var r,i,o=null,a=0,s=this[0];if(e===t){if(this.length&&(o=x.data(s),1===s.nodeType&&!x._data(s,"parsedAttrs"))){for(r=s.attributes;r.length>a;a++)i=r[a].name,0===i.indexOf("data-")&&(i=x.camelCase(i.slice(5)),$(s,i,o[i]));x._data(s,"parsedAttrs",!0)}return o}return"object"==typeof e?this.each(function(){x.data(this,e)}):arguments.length>1?this.each(function(){x.data(this,e,n)}):s?$(s,e,x.data(s,e)):null},removeData:function(e){return this.each(function(){x.removeData(this,e)})}});function $(e,n,r){if(r===t&&1===e.nodeType){var i="data-"+n.replace(P,"-$1").toLowerCase();if(r=e.getAttribute(i),"string"==typeof r){try{r="true"===r?!0:"false"===r?!1:"null"===r?null:+r+""===r?+r:B.test(r)?x.parseJSON(r):r}catch(o){}x.data(e,n,r)}else r=t}return r}function I(e){var t;for(t in e)if(("data"!==t||!x.isEmptyObject(e[t]))&&"toJSON"!==t)return!1;return!0}x.extend({queue:function(e,n,r){var i;return e?(n=(n||"fx")+"queue",i=x._data(e,n),r&&(!i||x.isArray(r)?i=x._data(e,n,x.makeArray(r)):i.push(r)),i||[]):t},dequeue:function(e,t){t=t||"fx";var n=x.queue(e,t),r=n.length,i=n.shift(),o=x._queueHooks(e,t),a=function(){x.dequeue(e,t)};"inprogress"===i&&(i=n.shift(),r--),i&&("fx"===t&&n.unshift("inprogress"),delete o.stop,i.call(e,a,o)),!r&&o&&o.empty.fire()},_queueHooks:function(e,t){var n=t+"queueHooks";return x._data(e,n)||x._data(e,n,{empty:x.Callbacks("once memory").add(function(){x._removeData(e,t+"queue"),x._removeData(e,n)})})}}),x.fn.extend({queue:function(e,n){var r=2;return"string"!=typeof e&&(n=e,e="fx",r--),r>arguments.length?x.queue(this[0],e):n===t?this:this.each(function(){var t=x.queue(this,e,n);x._queueHooks(this,e),"fx"===e&&"inprogress"!==t[0]&&x.dequeue(this,e)})},dequeue:function(e){return this.each(function(){x.dequeue(this,e)})},delay:function(e,t){return e=x.fx?x.fx.speeds[e]||e:e,t=t||"fx",this.queue(t,function(t,n){var r=setTimeout(t,e);n.stop=function(){clearTimeout(r)}})},clearQueue:function(e){return this.queue(e||"fx",[])},promise:function(e,n){var r,i=1,o=x.Deferred(),a=this,s=this.length,l=function(){--i||o.resolveWith(a,[a])};"string"!=typeof e&&(n=e,e=t),e=e||"fx";while(s--)r=x._data(a[s],e+"queueHooks"),r&&r.empty&&(i++,r.empty.add(l));return l(),o.promise(n)}});var z,X,U=/[\t\r\n\f]/g,V=/\r/g,Y=/^(?:input|select|textarea|button|object)$/i,J=/^(?:a|area)$/i,G=/^(?:checked|selected)$/i,Q=x.support.getSetAttribute,K=x.support.input;x.fn.extend({attr:function(e,t){return x.access(this,x.attr,e,t,arguments.length>1)},removeAttr:function(e){return this.each(function(){x.removeAttr(this,e)})},prop:function(e,t){return x.access(this,x.prop,e,t,arguments.length>1)},removeProp:function(e){return e=x.propFix[e]||e,this.each(function(){try{this[e]=t,delete this[e]}catch(n){}})},addClass:function(e){var t,n,r,i,o,a=0,s=this.length,l="string"==typeof e&&e;if(x.isFunction(e))return this.each(function(t){x(this).addClass(e.call(this,t,this.className))});if(l)for(t=(e||"").match(T)||[];s>a;a++)if(n=this[a],r=1===n.nodeType&&(n.className?(" "+n.className+" ").replace(U," "):" ")){o=0;while(i=t[o++])0>r.indexOf(" "+i+" ")&&(r+=i+" ");n.className=x.trim(r)}return this},removeClass:function(e){var t,n,r,i,o,a=0,s=this.length,l=0===arguments.length||"string"==typeof e&&e;if(x.isFunction(e))return this.each(function(t){x(this).removeClass(e.call(this,t,this.className))});if(l)for(t=(e||"").match(T)||[];s>a;a++)if(n=this[a],r=1===n.nodeType&&(n.className?(" "+n.className+" ").replace(U," "):"")){o=0;while(i=t[o++])while(r.indexOf(" "+i+" ")>=0)r=r.replace(" "+i+" "," ");n.className=e?x.trim(r):""}return this},toggleClass:function(e,t){var n=typeof e;return"boolean"==typeof t&&"string"===n?t?this.addClass(e):this.removeClass(e):x.isFunction(e)?this.each(function(n){x(this).toggleClass(e.call(this,n,this.className,t),t)}):this.each(function(){if("string"===n){var t,r=0,o=x(this),a=e.match(T)||[];while(t=a[r++])o.hasClass(t)?o.removeClass(t):o.addClass(t)}else(n===i||"boolean"===n)&&(this.className&&x._data(this,"__className__",this.className),this.className=this.className||e===!1?"":x._data(this,"__className__")||"")})},hasClass:function(e){var t=" "+e+" ",n=0,r=this.length;for(;r>n;n++)if(1===this[n].nodeType&&(" "+this[n].className+" ").replace(U," ").indexOf(t)>=0)return!0;return!1},val:function(e){var n,r,i,o=this[0];{if(arguments.length)return i=x.isFunction(e),this.each(function(n){var o;1===this.nodeType&&(o=i?e.call(this,n,x(this).val()):e,null==o?o="":"number"==typeof o?o+="":x.isArray(o)&&(o=x.map(o,function(e){return null==e?"":e+""})),r=x.valHooks[this.type]||x.valHooks[this.nodeName.toLowerCase()],r&&"set"in r&&r.set(this,o,"value")!==t||(this.value=o))});if(o)return r=x.valHooks[o.type]||x.valHooks[o.nodeName.toLowerCase()],r&&"get"in r&&(n=r.get(o,"value"))!==t?n:(n=o.value,"string"==typeof n?n.replace(V,""):null==n?"":n)}}}),x.extend({valHooks:{option:{get:function(e){var t=x.find.attr(e,"value");return null!=t?t:e.text}},select:{get:function(e){var t,n,r=e.options,i=e.selectedIndex,o="select-one"===e.type||0>i,a=o?null:[],s=o?i+1:r.length,l=0>i?s:o?i:0;for(;s>l;l++)if(n=r[l],!(!n.selected&&l!==i||(x.support.optDisabled?n.disabled:null!==n.getAttribute("disabled"))||n.parentNode.disabled&&x.nodeName(n.parentNode,"optgroup"))){if(t=x(n).val(),o)return t;a.push(t)}return a},set:function(e,t){var n,r,i=e.options,o=x.makeArray(t),a=i.length;while(a--)r=i[a],(r.selected=x.inArray(x(r).val(),o)>=0)&&(n=!0);return n||(e.selectedIndex=-1),o}}},attr:function(e,n,r){var o,a,s=e.nodeType;if(e&&3!==s&&8!==s&&2!==s)return typeof e.getAttribute===i?x.prop(e,n,r):(1===s&&x.isXMLDoc(e)||(n=n.toLowerCase(),o=x.attrHooks[n]||(x.expr.match.bool.test(n)?X:z)),r===t?o&&"get"in o&&null!==(a=o.get(e,n))?a:(a=x.find.attr(e,n),null==a?t:a):null!==r?o&&"set"in o&&(a=o.set(e,r,n))!==t?a:(e.setAttribute(n,r+""),r):(x.removeAttr(e,n),t))},removeAttr:function(e,t){var n,r,i=0,o=t&&t.match(T);if(o&&1===e.nodeType)while(n=o[i++])r=x.propFix[n]||n,x.expr.match.bool.test(n)?K&&Q||!G.test(n)?e[r]=!1:e[x.camelCase("default-"+n)]=e[r]=!1:x.attr(e,n,""),e.removeAttribute(Q?n:r)},attrHooks:{type:{set:function(e,t){if(!x.support.radioValue&&"radio"===t&&x.nodeName(e,"input")){var n=e.value;return e.setAttribute("type",t),n&&(e.value=n),t}}}},propFix:{"for":"htmlFor","class":"className"},prop:function(e,n,r){var i,o,a,s=e.nodeType;if(e&&3!==s&&8!==s&&2!==s)return a=1!==s||!x.isXMLDoc(e),a&&(n=x.propFix[n]||n,o=x.propHooks[n]),r!==t?o&&"set"in o&&(i=o.set(e,r,n))!==t?i:e[n]=r:o&&"get"in o&&null!==(i=o.get(e,n))?i:e[n]},propHooks:{tabIndex:{get:function(e){var t=x.find.attr(e,"tabindex");return t?parseInt(t,10):Y.test(e.nodeName)||J.test(e.nodeName)&&e.href?0:-1}}}}),X={set:function(e,t,n){return t===!1?x.removeAttr(e,n):K&&Q||!G.test(n)?e.setAttribute(!Q&&x.propFix[n]||n,n):e[x.camelCase("default-"+n)]=e[n]=!0,n}},x.each(x.expr.match.bool.source.match(/\w+/g),function(e,n){var r=x.expr.attrHandle[n]||x.find.attr;x.expr.attrHandle[n]=K&&Q||!G.test(n)?function(e,n,i){var o=x.expr.attrHandle[n],a=i?t:(x.expr.attrHandle[n]=t)!=r(e,n,i)?n.toLowerCase():null;return x.expr.attrHandle[n]=o,a}:function(e,n,r){return r?t:e[x.camelCase("default-"+n)]?n.toLowerCase():null}}),K&&Q||(x.attrHooks.value={set:function(e,n,r){return x.nodeName(e,"input")?(e.defaultValue=n,t):z&&z.set(e,n,r)}}),Q||(z={set:function(e,n,r){var i=e.getAttributeNode(r);return i||e.setAttributeNode(i=e.ownerDocument.createAttribute(r)),i.value=n+="","value"===r||n===e.getAttribute(r)?n:t}},x.expr.attrHandle.id=x.expr.attrHandle.name=x.expr.attrHandle.coords=function(e,n,r){var i;return r?t:(i=e.getAttributeNode(n))&&""!==i.value?i.value:null},x.valHooks.button={get:function(e,n){var r=e.getAttributeNode(n);return r&&r.specified?r.value:t},set:z.set},x.attrHooks.contenteditable={set:function(e,t,n){z.set(e,""===t?!1:t,n)}},x.each(["width","height"],function(e,n){x.attrHooks[n]={set:function(e,r){return""===r?(e.setAttribute(n,"auto"),r):t}}})),x.support.hrefNormalized||x.each(["href","src"],function(e,t){x.propHooks[t]={get:function(e){return e.getAttribute(t,4)}}}),x.support.style||(x.attrHooks.style={get:function(e){return e.style.cssText||t},set:function(e,t){return e.style.cssText=t+""}}),x.support.optSelected||(x.propHooks.selected={get:function(e){var t=e.parentNode;return t&&(t.selectedIndex,t.parentNode&&t.parentNode.selectedIndex),null}}),x.each(["tabIndex","readOnly","maxLength","cellSpacing","cellPadding","rowSpan","colSpan","useMap","frameBorder","contentEditable"],function(){x.propFix[this.toLowerCase()]=this}),x.support.enctype||(x.propFix.enctype="encoding"),x.each(["radio","checkbox"],function(){x.valHooks[this]={set:function(e,n){return x.isArray(n)?e.checked=x.inArray(x(e).val(),n)>=0:t}},x.support.checkOn||(x.valHooks[this].get=function(e){return null===e.getAttribute("value")?"on":e.value})});var Z=/^(?:input|select|textarea)$/i,et=/^key/,tt=/^(?:mouse|contextmenu)|click/,nt=/^(?:focusinfocus|focusoutblur)$/,rt=/^([^.]*)(?:\.(.+)|)$/;function it(){return!0}function ot(){return!1}function at(){try{return a.activeElement}catch(e){}}x.event={global:{},add:function(e,n,r,o,a){var s,l,u,c,p,f,d,h,g,m,y,v=x._data(e);if(v){r.handler&&(c=r,r=c.handler,a=c.selector),r.guid||(r.guid=x.guid++),(l=v.events)||(l=v.events={}),(f=v.handle)||(f=v.handle=function(e){return typeof x===i||e&&x.event.triggered===e.type?t:x.event.dispatch.apply(f.elem,arguments)},f.elem=e),n=(n||"").match(T)||[""],u=n.length;while(u--)s=rt.exec(n[u])||[],g=y=s[1],m=(s[2]||"").split(".").sort(),g&&(p=x.event.special[g]||{},g=(a?p.delegateType:p.bindType)||g,p=x.event.special[g]||{},d=x.extend({type:g,origType:y,data:o,handler:r,guid:r.guid,selector:a,needsContext:a&&x.expr.match.needsContext.test(a),namespace:m.join(".")},c),(h=l[g])||(h=l[g]=[],h.delegateCount=0,p.setup&&p.setup.call(e,o,m,f)!==!1||(e.addEventListener?e.addEventListener(g,f,!1):e.attachEvent&&e.attachEvent("on"+g,f))),p.add&&(p.add.call(e,d),d.handler.guid||(d.handler.guid=r.guid)),a?h.splice(h.delegateCount++,0,d):h.push(d),x.event.global[g]=!0);e=null}},remove:function(e,t,n,r,i){var o,a,s,l,u,c,p,f,d,h,g,m=x.hasData(e)&&x._data(e);if(m&&(c=m.events)){t=(t||"").match(T)||[""],u=t.length;while(u--)if(s=rt.exec(t[u])||[],d=g=s[1],h=(s[2]||"").split(".").sort(),d){p=x.event.special[d]||{},d=(r?p.delegateType:p.bindType)||d,f=c[d]||[],s=s[2]&&RegExp("(^|\\.)"+h.join("\\.(?:.*\\.|)")+"(\\.|$)"),l=o=f.length;while(o--)a=f[o],!i&&g!==a.origType||n&&n.guid!==a.guid||s&&!s.test(a.namespace)||r&&r!==a.selector&&("**"!==r||!a.selector)||(f.splice(o,1),a.selector&&f.delegateCount--,p.remove&&p.remove.call(e,a));l&&!f.length&&(p.teardown&&p.teardown.call(e,h,m.handle)!==!1||x.removeEvent(e,d,m.handle),delete c[d])}else for(d in c)x.event.remove(e,d+t[u],n,r,!0);x.isEmptyObject(c)&&(delete m.handle,x._removeData(e,"events"))}},trigger:function(n,r,i,o){var s,l,u,c,p,f,d,h=[i||a],g=v.call(n,"type")?n.type:n,m=v.call(n,"namespace")?n.namespace.split("."):[];if(u=f=i=i||a,3!==i.nodeType&&8!==i.nodeType&&!nt.test(g+x.event.triggered)&&(g.indexOf(".")>=0&&(m=g.split("."),g=m.shift(),m.sort()),l=0>g.indexOf(":")&&"on"+g,n=n[x.expando]?n:new x.Event(g,"object"==typeof n&&n),n.isTrigger=o?2:3,n.namespace=m.join("."),n.namespace_re=n.namespace?RegExp("(^|\\.)"+m.join("\\.(?:.*\\.|)")+"(\\.|$)"):null,n.result=t,n.target||(n.target=i),r=null==r?[n]:x.makeArray(r,[n]),p=x.event.special[g]||{},o||!p.trigger||p.trigger.apply(i,r)!==!1)){if(!o&&!p.noBubble&&!x.isWindow(i)){for(c=p.delegateType||g,nt.test(c+g)||(u=u.parentNode);u;u=u.parentNode)h.push(u),f=u;f===(i.ownerDocument||a)&&h.push(f.defaultView||f.parentWindow||e)}d=0;while((u=h[d++])&&!n.isPropagationStopped())n.type=d>1?c:p.bindType||g,s=(x._data(u,"events")||{})[n.type]&&x._data(u,"handle"),s&&s.apply(u,r),s=l&&u[l],s&&x.acceptData(u)&&s.apply&&s.apply(u,r)===!1&&n.preventDefault();if(n.type=g,!o&&!n.isDefaultPrevented()&&(!p._default||p._default.apply(h.pop(),r)===!1)&&x.acceptData(i)&&l&&i[g]&&!x.isWindow(i)){f=i[l],f&&(i[l]=null),x.event.triggered=g;try{i[g]()}catch(y){}x.event.triggered=t,f&&(i[l]=f)}return n.result}},dispatch:function(e){e=x.event.fix(e);var n,r,i,o,a,s=[],l=g.call(arguments),u=(x._data(this,"events")||{})[e.type]||[],c=x.event.special[e.type]||{};if(l[0]=e,e.delegateTarget=this,!c.preDispatch||c.preDispatch.call(this,e)!==!1){s=x.event.handlers.call(this,e,u),n=0;while((o=s[n++])&&!e.isPropagationStopped()){e.currentTarget=o.elem,a=0;while((i=o.handlers[a++])&&!e.isImmediatePropagationStopped())(!e.namespace_re||e.namespace_re.test(i.namespace))&&(e.handleObj=i,e.data=i.data,r=((x.event.special[i.origType]||{}).handle||i.handler).apply(o.elem,l),r!==t&&(e.result=r)===!1&&(e.preventDefault(),e.stopPropagation()))}return c.postDispatch&&c.postDispatch.call(this,e),e.result}},handlers:function(e,n){var r,i,o,a,s=[],l=n.delegateCount,u=e.target;if(l&&u.nodeType&&(!e.button||"click"!==e.type))for(;u!=this;u=u.parentNode||this)if(1===u.nodeType&&(u.disabled!==!0||"click"!==e.type)){for(o=[],a=0;l>a;a++)i=n[a],r=i.selector+" ",o[r]===t&&(o[r]=i.needsContext?x(r,this).index(u)>=0:x.find(r,this,null,[u]).length),o[r]&&o.push(i);o.length&&s.push({elem:u,handlers:o})}return n.length>l&&s.push({elem:this,handlers:n.slice(l)}),s},fix:function(e){if(e[x.expando])return e;var t,n,r,i=e.type,o=e,s=this.fixHooks[i];s||(this.fixHooks[i]=s=tt.test(i)?this.mouseHooks:et.test(i)?this.keyHooks:{}),r=s.props?this.props.concat(s.props):this.props,e=new x.Event(o),t=r.length;while(t--)n=r[t],e[n]=o[n];return e.target||(e.target=o.srcElement||a),3===e.target.nodeType&&(e.target=e.target.parentNode),e.metaKey=!!e.metaKey,s.filter?s.filter(e,o):e},props:"altKey bubbles cancelable ctrlKey currentTarget eventPhase metaKey relatedTarget shiftKey target timeStamp view which".split(" "),fixHooks:{},keyHooks:{props:"char charCode key keyCode".split(" "),filter:function(e,t){return null==e.which&&(e.which=null!=t.charCode?t.charCode:t.keyCode),e}},mouseHooks:{props:"button buttons clientX clientY fromElement offsetX offsetY pageX pageY screenX screenY toElement".split(" "),filter:function(e,n){var r,i,o,s=n.button,l=n.fromElement;return null==e.pageX&&null!=n.clientX&&(i=e.target.ownerDocument||a,o=i.documentElement,r=i.body,e.pageX=n.clientX+(o&&o.scrollLeft||r&&r.scrollLeft||0)-(o&&o.clientLeft||r&&r.clientLeft||0),e.pageY=n.clientY+(o&&o.scrollTop||r&&r.scrollTop||0)-(o&&o.clientTop||r&&r.clientTop||0)),!e.relatedTarget&&l&&(e.relatedTarget=l===e.target?n.toElement:l),e.which||s===t||(e.which=1&s?1:2&s?3:4&s?2:0),e}},special:{load:{noBubble:!0},focus:{trigger:function(){if(this!==at()&&this.focus)try{return this.focus(),!1}catch(e){}},delegateType:"focusin"},blur:{trigger:function(){return this===at()&&this.blur?(this.blur(),!1):t},delegateType:"focusout"},click:{trigger:function(){return x.nodeName(this,"input")&&"checkbox"===this.type&&this.click?(this.click(),!1):t},_default:function(e){return x.nodeName(e.target,"a")}},beforeunload:{postDispatch:function(e){e.result!==t&&(e.originalEvent.returnValue=e.result)}}},simulate:function(e,t,n,r){var i=x.extend(new x.Event,n,{type:e,isSimulated:!0,originalEvent:{}});r?x.event.trigger(i,null,t):x.event.dispatch.call(t,i),i.isDefaultPrevented()&&n.preventDefault()}},x.removeEvent=a.removeEventListener?function(e,t,n){e.removeEventListener&&e.removeEventListener(t,n,!1)}:function(e,t,n){var r="on"+t;e.detachEvent&&(typeof e[r]===i&&(e[r]=null),e.detachEvent(r,n))},x.Event=function(e,n){return this instanceof x.Event?(e&&e.type?(this.originalEvent=e,this.type=e.type,this.isDefaultPrevented=e.defaultPrevented||e.returnValue===!1||e.getPreventDefault&&e.getPreventDefault()?it:ot):this.type=e,n&&x.extend(this,n),this.timeStamp=e&&e.timeStamp||x.now(),this[x.expando]=!0,t):new x.Event(e,n)},x.Event.prototype={isDefaultPrevented:ot,isPropagationStopped:ot,isImmediatePropagationStopped:ot,preventDefault:function(){var e=this.originalEvent;this.isDefaultPrevented=it,e&&(e.preventDefault?e.preventDefault():e.returnValue=!1)},stopPropagation:function(){var e=this.originalEvent;this.isPropagationStopped=it,e&&(e.stopPropagation&&e.stopPropagation(),e.cancelBubble=!0)},stopImmediatePropagation:function(){this.isImmediatePropagationStopped=it,this.stopPropagation()}},x.each({mouseenter:"mouseover",mouseleave:"mouseout"},function(e,t){x.event.special[e]={delegateType:t,bindType:t,handle:function(e){var n,r=this,i=e.relatedTarget,o=e.handleObj;return(!i||i!==r&&!x.contains(r,i))&&(e.type=o.origType,n=o.handler.apply(this,arguments),e.type=t),n}}}),x.support.submitBubbles||(x.event.special.submit={setup:function(){return x.nodeName(this,"form")?!1:(x.event.add(this,"click._submit keypress._submit",function(e){var n=e.target,r=x.nodeName(n,"input")||x.nodeName(n,"button")?n.form:t;r&&!x._data(r,"submitBubbles")&&(x.event.add(r,"submit._submit",function(e){e._submit_bubble=!0}),x._data(r,"submitBubbles",!0))}),t)},postDispatch:function(e){e._submit_bubble&&(delete e._submit_bubble,this.parentNode&&!e.isTrigger&&x.event.simulate("submit",this.parentNode,e,!0))},teardown:function(){return x.nodeName(this,"form")?!1:(x.event.remove(this,"._submit"),t)}}),x.support.changeBubbles||(x.event.special.change={setup:function(){return Z.test(this.nodeName)?(("checkbox"===this.type||"radio"===this.type)&&(x.event.add(this,"propertychange._change",function(e){"checked"===e.originalEvent.propertyName&&(this._just_changed=!0)}),x.event.add(this,"click._change",function(e){this._just_changed&&!e.isTrigger&&(this._just_changed=!1),x.event.simulate("change",this,e,!0)})),!1):(x.event.add(this,"beforeactivate._change",function(e){var t=e.target;Z.test(t.nodeName)&&!x._data(t,"changeBubbles")&&(x.event.add(t,"change._change",function(e){!this.parentNode||e.isSimulated||e.isTrigger||x.event.simulate("change",this.parentNode,e,!0)}),x._data(t,"changeBubbles",!0))}),t)},handle:function(e){var n=e.target;return this!==n||e.isSimulated||e.isTrigger||"radio"!==n.type&&"checkbox"!==n.type?e.handleObj.handler.apply(this,arguments):t},teardown:function(){return x.event.remove(this,"._change"),!Z.test(this.nodeName)}}),x.support.focusinBubbles||x.each({focus:"focusin",blur:"focusout"},function(e,t){var n=0,r=function(e){x.event.simulate(t,e.target,x.event.fix(e),!0)};x.event.special[t]={setup:function(){0===n++&&a.addEventListener(e,r,!0)},teardown:function(){0===--n&&a.removeEventListener(e,r,!0)}}}),x.fn.extend({on:function(e,n,r,i,o){var a,s;if("object"==typeof e){"string"!=typeof n&&(r=r||n,n=t);for(a in e)this.on(a,n,r,e[a],o);return this}if(null==r&&null==i?(i=n,r=n=t):null==i&&("string"==typeof n?(i=r,r=t):(i=r,r=n,n=t)),i===!1)i=ot;else if(!i)return this;return 1===o&&(s=i,i=function(e){return x().off(e),s.apply(this,arguments)},i.guid=s.guid||(s.guid=x.guid++)),this.each(function(){x.event.add(this,e,i,r,n)})},one:function(e,t,n,r){return this.on(e,t,n,r,1)},off:function(e,n,r){var i,o;if(e&&e.preventDefault&&e.handleObj)return i=e.handleObj,x(e.delegateTarget).off(i.namespace?i.origType+"."+i.namespace:i.origType,i.selector,i.handler),this;if("object"==typeof e){for(o in e)this.off(o,n,e[o]);return this}return(n===!1||"function"==typeof n)&&(r=n,n=t),r===!1&&(r=ot),this.each(function(){x.event.remove(this,e,r,n)})},trigger:function(e,t){return this.each(function(){x.event.trigger(e,t,this)})},triggerHandler:function(e,n){var r=this[0];return r?x.event.trigger(e,n,r,!0):t}});var st=/^.[^:#\[\.,]*$/,lt=/^(?:parents|prev(?:Until|All))/,ut=x.expr.match.needsContext,ct={children:!0,contents:!0,next:!0,prev:!0};x.fn.extend({find:function(e){var t,n=[],r=this,i=r.length;if("string"!=typeof e)return this.pushStack(x(e).filter(function(){for(t=0;i>t;t++)if(x.contains(r[t],this))return!0}));for(t=0;i>t;t++)x.find(e,r[t],n);return n=this.pushStack(i>1?x.unique(n):n),n.selector=this.selector?this.selector+" "+e:e,n},has:function(e){var t,n=x(e,this),r=n.length;return this.filter(function(){for(t=0;r>t;t++)if(x.contains(this,n[t]))return!0})},not:function(e){return this.pushStack(ft(this,e||[],!0))},filter:function(e){return this.pushStack(ft(this,e||[],!1))},is:function(e){return!!ft(this,"string"==typeof e&&ut.test(e)?x(e):e||[],!1).length},closest:function(e,t){var n,r=0,i=this.length,o=[],a=ut.test(e)||"string"!=typeof e?x(e,t||this.context):0;for(;i>r;r++)for(n=this[r];n&&n!==t;n=n.parentNode)if(11>n.nodeType&&(a?a.index(n)>-1:1===n.nodeType&&x.find.matchesSelector(n,e))){n=o.push(n);break}return this.pushStack(o.length>1?x.unique(o):o)},index:function(e){return e?"string"==typeof e?x.inArray(this[0],x(e)):x.inArray(e.jquery?e[0]:e,this):this[0]&&this[0].parentNode?this.first().prevAll().length:-1},add:function(e,t){var n="string"==typeof e?x(e,t):x.makeArray(e&&e.nodeType?[e]:e),r=x.merge(this.get(),n);return this.pushStack(x.unique(r))},addBack:function(e){return this.add(null==e?this.prevObject:this.prevObject.filter(e))}});function pt(e,t){do e=e[t];while(e&&1!==e.nodeType);return e}x.each({parent:function(e){var t=e.parentNode;return t&&11!==t.nodeType?t:null},parents:function(e){return x.dir(e,"parentNode")},parentsUntil:function(e,t,n){return x.dir(e,"parentNode",n)},next:function(e){return pt(e,"nextSibling")},prev:function(e){return pt(e,"previousSibling")},nextAll:function(e){return x.dir(e,"nextSibling")},prevAll:function(e){return x.dir(e,"previousSibling")},nextUntil:function(e,t,n){return x.dir(e,"nextSibling",n)},prevUntil:function(e,t,n){return x.dir(e,"previousSibling",n)},siblings:function(e){return x.sibling((e.parentNode||{}).firstChild,e)},children:function(e){return x.sibling(e.firstChild)},contents:function(e){return x.nodeName(e,"iframe")?e.contentDocument||e.contentWindow.document:x.merge([],e.childNodes)}},function(e,t){x.fn[e]=function(n,r){var i=x.map(this,t,n);return"Until"!==e.slice(-5)&&(r=n),r&&"string"==typeof r&&(i=x.filter(r,i)),this.length>1&&(ct[e]||(i=x.unique(i)),lt.test(e)&&(i=i.reverse())),this.pushStack(i)}}),x.extend({filter:function(e,t,n){var r=t[0];return n&&(e=":not("+e+")"),1===t.length&&1===r.nodeType?x.find.matchesSelector(r,e)?[r]:[]:x.find.matches(e,x.grep(t,function(e){return 1===e.nodeType}))},dir:function(e,n,r){var i=[],o=e[n];while(o&&9!==o.nodeType&&(r===t||1!==o.nodeType||!x(o).is(r)))1===o.nodeType&&i.push(o),o=o[n];return i},sibling:function(e,t){var n=[];for(;e;e=e.nextSibling)1===e.nodeType&&e!==t&&n.push(e);return n}});function ft(e,t,n){if(x.isFunction(t))return x.grep(e,function(e,r){return!!t.call(e,r,e)!==n});if(t.nodeType)return x.grep(e,function(e){return e===t!==n});if("string"==typeof t){if(st.test(t))return x.filter(t,e,n);t=x.filter(t,e)}return x.grep(e,function(e){return x.inArray(e,t)>=0!==n})}function dt(e){var t=ht.split("|"),n=e.createDocumentFragment();if(n.createElement)while(t.length)n.createElement(t.pop());return n}var ht="abbr|article|aside|audio|bdi|canvas|data|datalist|details|figcaption|figure|footer|header|hgroup|mark|meter|nav|output|progress|section|summary|time|video",gt=/ jQuery\d+="(?:null|\d+)"/g,mt=RegExp("<(?:"+ht+")[\\s/>]","i"),yt=/^\s+/,vt=/<(?!area|br|col|embed|hr|img|input|link|meta|param)(([\w:]+)[^>]*)\/>/gi,bt=/<([\w:]+)/,xt=/\s*$/g,At={option:[1,""],legend:[1,"
","
"],area:[1,"",""],param:[1,"",""],thead:[1,"","
"],tr:[2,"","
"],col:[2,"","
"],td:[3,"","
"],_default:x.support.htmlSerialize?[0,"",""]:[1,"X
","
"]},jt=dt(a),Dt=jt.appendChild(a.createElement("div"));At.optgroup=At.option,At.tbody=At.tfoot=At.colgroup=At.caption=At.thead,At.th=At.td,x.fn.extend({text:function(e){return x.access(this,function(e){return e===t?x.text(this):this.empty().append((this[0]&&this[0].ownerDocument||a).createTextNode(e))},null,e,arguments.length)},append:function(){return this.domManip(arguments,function(e){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var t=Lt(this,e);t.appendChild(e)}})},prepend:function(){return this.domManip(arguments,function(e){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var t=Lt(this,e);t.insertBefore(e,t.firstChild)}})},before:function(){return this.domManip(arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this)})},after:function(){return this.domManip(arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this.nextSibling)})},remove:function(e,t){var n,r=e?x.filter(e,this):this,i=0;for(;null!=(n=r[i]);i++)t||1!==n.nodeType||x.cleanData(Ft(n)),n.parentNode&&(t&&x.contains(n.ownerDocument,n)&&_t(Ft(n,"script")),n.parentNode.removeChild(n));return this},empty:function(){var e,t=0;for(;null!=(e=this[t]);t++){1===e.nodeType&&x.cleanData(Ft(e,!1));while(e.firstChild)e.removeChild(e.firstChild);e.options&&x.nodeName(e,"select")&&(e.options.length=0)}return this},clone:function(e,t){return e=null==e?!1:e,t=null==t?e:t,this.map(function(){return x.clone(this,e,t)})},html:function(e){return x.access(this,function(e){var n=this[0]||{},r=0,i=this.length;if(e===t)return 1===n.nodeType?n.innerHTML.replace(gt,""):t;if(!("string"!=typeof e||Tt.test(e)||!x.support.htmlSerialize&&mt.test(e)||!x.support.leadingWhitespace&&yt.test(e)||At[(bt.exec(e)||["",""])[1].toLowerCase()])){e=e.replace(vt,"<$1>");try{for(;i>r;r++)n=this[r]||{},1===n.nodeType&&(x.cleanData(Ft(n,!1)),n.innerHTML=e);n=0}catch(o){}}n&&this.empty().append(e)},null,e,arguments.length)},replaceWith:function(){var e=x.map(this,function(e){return[e.nextSibling,e.parentNode]}),t=0;return this.domManip(arguments,function(n){var r=e[t++],i=e[t++];i&&(r&&r.parentNode!==i&&(r=this.nextSibling),x(this).remove(),i.insertBefore(n,r))},!0),t?this:this.remove()},detach:function(e){return this.remove(e,!0)},domManip:function(e,t,n){e=d.apply([],e);var r,i,o,a,s,l,u=0,c=this.length,p=this,f=c-1,h=e[0],g=x.isFunction(h);if(g||!(1>=c||"string"!=typeof h||x.support.checkClone)&&Nt.test(h))return this.each(function(r){var i=p.eq(r);g&&(e[0]=h.call(this,r,i.html())),i.domManip(e,t,n)});if(c&&(l=x.buildFragment(e,this[0].ownerDocument,!1,!n&&this),r=l.firstChild,1===l.childNodes.length&&(l=r),r)){for(a=x.map(Ft(l,"script"),Ht),o=a.length;c>u;u++)i=l,u!==f&&(i=x.clone(i,!0,!0),o&&x.merge(a,Ft(i,"script"))),t.call(this[u],i,u);if(o)for(s=a[a.length-1].ownerDocument,x.map(a,qt),u=0;o>u;u++)i=a[u],kt.test(i.type||"")&&!x._data(i,"globalEval")&&x.contains(s,i)&&(i.src?x._evalUrl(i.src):x.globalEval((i.text||i.textContent||i.innerHTML||"").replace(St,"")));l=r=null}return this}});function Lt(e,t){return x.nodeName(e,"table")&&x.nodeName(1===t.nodeType?t:t.firstChild,"tr")?e.getElementsByTagName("tbody")[0]||e.appendChild(e.ownerDocument.createElement("tbody")):e}function Ht(e){return e.type=(null!==x.find.attr(e,"type"))+"/"+e.type,e}function qt(e){var t=Et.exec(e.type);return t?e.type=t[1]:e.removeAttribute("type"),e}function _t(e,t){var n,r=0;for(;null!=(n=e[r]);r++)x._data(n,"globalEval",!t||x._data(t[r],"globalEval"))}function Mt(e,t){if(1===t.nodeType&&x.hasData(e)){var n,r,i,o=x._data(e),a=x._data(t,o),s=o.events;if(s){delete a.handle,a.events={};for(n in s)for(r=0,i=s[n].length;i>r;r++)x.event.add(t,n,s[n][r])}a.data&&(a.data=x.extend({},a.data))}}function Ot(e,t){var n,r,i;if(1===t.nodeType){if(n=t.nodeName.toLowerCase(),!x.support.noCloneEvent&&t[x.expando]){i=x._data(t);for(r in i.events)x.removeEvent(t,r,i.handle);t.removeAttribute(x.expando)}"script"===n&&t.text!==e.text?(Ht(t).text=e.text,qt(t)):"object"===n?(t.parentNode&&(t.outerHTML=e.outerHTML),x.support.html5Clone&&e.innerHTML&&!x.trim(t.innerHTML)&&(t.innerHTML=e.innerHTML)):"input"===n&&Ct.test(e.type)?(t.defaultChecked=t.checked=e.checked,t.value!==e.value&&(t.value=e.value)):"option"===n?t.defaultSelected=t.selected=e.defaultSelected:("input"===n||"textarea"===n)&&(t.defaultValue=e.defaultValue)}}x.each({appendTo:"append",prependTo:"prepend",insertBefore:"before",insertAfter:"after",replaceAll:"replaceWith"},function(e,t){x.fn[e]=function(e){var n,r=0,i=[],o=x(e),a=o.length-1;for(;a>=r;r++)n=r===a?this:this.clone(!0),x(o[r])[t](n),h.apply(i,n.get());return this.pushStack(i)}});function Ft(e,n){var r,o,a=0,s=typeof e.getElementsByTagName!==i?e.getElementsByTagName(n||"*"):typeof e.querySelectorAll!==i?e.querySelectorAll(n||"*"):t;if(!s)for(s=[],r=e.childNodes||e;null!=(o=r[a]);a++)!n||x.nodeName(o,n)?s.push(o):x.merge(s,Ft(o,n));return n===t||n&&x.nodeName(e,n)?x.merge([e],s):s}function Bt(e){Ct.test(e.type)&&(e.defaultChecked=e.checked)}x.extend({clone:function(e,t,n){var r,i,o,a,s,l=x.contains(e.ownerDocument,e);if(x.support.html5Clone||x.isXMLDoc(e)||!mt.test("<"+e.nodeName+">")?o=e.cloneNode(!0):(Dt.innerHTML=e.outerHTML,Dt.removeChild(o=Dt.firstChild)),!(x.support.noCloneEvent&&x.support.noCloneChecked||1!==e.nodeType&&11!==e.nodeType||x.isXMLDoc(e)))for(r=Ft(o),s=Ft(e),a=0;null!=(i=s[a]);++a)r[a]&&Ot(i,r[a]);if(t)if(n)for(s=s||Ft(e),r=r||Ft(o),a=0;null!=(i=s[a]);a++)Mt(i,r[a]);else Mt(e,o);return r=Ft(o,"script"),r.length>0&&_t(r,!l&&Ft(e,"script")),r=s=i=null,o},buildFragment:function(e,t,n,r){var i,o,a,s,l,u,c,p=e.length,f=dt(t),d=[],h=0;for(;p>h;h++)if(o=e[h],o||0===o)if("object"===x.type(o))x.merge(d,o.nodeType?[o]:o);else if(wt.test(o)){s=s||f.appendChild(t.createElement("div")),l=(bt.exec(o)||["",""])[1].toLowerCase(),c=At[l]||At._default,s.innerHTML=c[1]+o.replace(vt,"<$1>")+c[2],i=c[0];while(i--)s=s.lastChild;if(!x.support.leadingWhitespace&&yt.test(o)&&d.push(t.createTextNode(yt.exec(o)[0])),!x.support.tbody){o="table"!==l||xt.test(o)?""!==c[1]||xt.test(o)?0:s:s.firstChild,i=o&&o.childNodes.length;while(i--)x.nodeName(u=o.childNodes[i],"tbody")&&!u.childNodes.length&&o.removeChild(u)}x.merge(d,s.childNodes),s.textContent="";while(s.firstChild)s.removeChild(s.firstChild);s=f.lastChild}else d.push(t.createTextNode(o));s&&f.removeChild(s),x.support.appendChecked||x.grep(Ft(d,"input"),Bt),h=0;while(o=d[h++])if((!r||-1===x.inArray(o,r))&&(a=x.contains(o.ownerDocument,o),s=Ft(f.appendChild(o),"script"),a&&_t(s),n)){i=0;while(o=s[i++])kt.test(o.type||"")&&n.push(o)}return s=null,f},cleanData:function(e,t){var n,r,o,a,s=0,l=x.expando,u=x.cache,c=x.support.deleteExpando,f=x.event.special;for(;null!=(n=e[s]);s++)if((t||x.acceptData(n))&&(o=n[l],a=o&&u[o])){if(a.events)for(r in a.events)f[r]?x.event.remove(n,r):x.removeEvent(n,r,a.handle); +u[o]&&(delete u[o],c?delete n[l]:typeof n.removeAttribute!==i?n.removeAttribute(l):n[l]=null,p.push(o))}},_evalUrl:function(e){return x.ajax({url:e,type:"GET",dataType:"script",async:!1,global:!1,"throws":!0})}}),x.fn.extend({wrapAll:function(e){if(x.isFunction(e))return this.each(function(t){x(this).wrapAll(e.call(this,t))});if(this[0]){var t=x(e,this[0].ownerDocument).eq(0).clone(!0);this[0].parentNode&&t.insertBefore(this[0]),t.map(function(){var e=this;while(e.firstChild&&1===e.firstChild.nodeType)e=e.firstChild;return e}).append(this)}return this},wrapInner:function(e){return x.isFunction(e)?this.each(function(t){x(this).wrapInner(e.call(this,t))}):this.each(function(){var t=x(this),n=t.contents();n.length?n.wrapAll(e):t.append(e)})},wrap:function(e){var t=x.isFunction(e);return this.each(function(n){x(this).wrapAll(t?e.call(this,n):e)})},unwrap:function(){return this.parent().each(function(){x.nodeName(this,"body")||x(this).replaceWith(this.childNodes)}).end()}});var Pt,Rt,Wt,$t=/alpha\([^)]*\)/i,It=/opacity\s*=\s*([^)]*)/,zt=/^(top|right|bottom|left)$/,Xt=/^(none|table(?!-c[ea]).+)/,Ut=/^margin/,Vt=RegExp("^("+w+")(.*)$","i"),Yt=RegExp("^("+w+")(?!px)[a-z%]+$","i"),Jt=RegExp("^([+-])=("+w+")","i"),Gt={BODY:"block"},Qt={position:"absolute",visibility:"hidden",display:"block"},Kt={letterSpacing:0,fontWeight:400},Zt=["Top","Right","Bottom","Left"],en=["Webkit","O","Moz","ms"];function tn(e,t){if(t in e)return t;var n=t.charAt(0).toUpperCase()+t.slice(1),r=t,i=en.length;while(i--)if(t=en[i]+n,t in e)return t;return r}function nn(e,t){return e=t||e,"none"===x.css(e,"display")||!x.contains(e.ownerDocument,e)}function rn(e,t){var n,r,i,o=[],a=0,s=e.length;for(;s>a;a++)r=e[a],r.style&&(o[a]=x._data(r,"olddisplay"),n=r.style.display,t?(o[a]||"none"!==n||(r.style.display=""),""===r.style.display&&nn(r)&&(o[a]=x._data(r,"olddisplay",ln(r.nodeName)))):o[a]||(i=nn(r),(n&&"none"!==n||!i)&&x._data(r,"olddisplay",i?n:x.css(r,"display"))));for(a=0;s>a;a++)r=e[a],r.style&&(t&&"none"!==r.style.display&&""!==r.style.display||(r.style.display=t?o[a]||"":"none"));return e}x.fn.extend({css:function(e,n){return x.access(this,function(e,n,r){var i,o,a={},s=0;if(x.isArray(n)){for(o=Rt(e),i=n.length;i>s;s++)a[n[s]]=x.css(e,n[s],!1,o);return a}return r!==t?x.style(e,n,r):x.css(e,n)},e,n,arguments.length>1)},show:function(){return rn(this,!0)},hide:function(){return rn(this)},toggle:function(e){return"boolean"==typeof e?e?this.show():this.hide():this.each(function(){nn(this)?x(this).show():x(this).hide()})}}),x.extend({cssHooks:{opacity:{get:function(e,t){if(t){var n=Wt(e,"opacity");return""===n?"1":n}}}},cssNumber:{columnCount:!0,fillOpacity:!0,fontWeight:!0,lineHeight:!0,opacity:!0,order:!0,orphans:!0,widows:!0,zIndex:!0,zoom:!0},cssProps:{"float":x.support.cssFloat?"cssFloat":"styleFloat"},style:function(e,n,r,i){if(e&&3!==e.nodeType&&8!==e.nodeType&&e.style){var o,a,s,l=x.camelCase(n),u=e.style;if(n=x.cssProps[l]||(x.cssProps[l]=tn(u,l)),s=x.cssHooks[n]||x.cssHooks[l],r===t)return s&&"get"in s&&(o=s.get(e,!1,i))!==t?o:u[n];if(a=typeof r,"string"===a&&(o=Jt.exec(r))&&(r=(o[1]+1)*o[2]+parseFloat(x.css(e,n)),a="number"),!(null==r||"number"===a&&isNaN(r)||("number"!==a||x.cssNumber[l]||(r+="px"),x.support.clearCloneStyle||""!==r||0!==n.indexOf("background")||(u[n]="inherit"),s&&"set"in s&&(r=s.set(e,r,i))===t)))try{u[n]=r}catch(c){}}},css:function(e,n,r,i){var o,a,s,l=x.camelCase(n);return n=x.cssProps[l]||(x.cssProps[l]=tn(e.style,l)),s=x.cssHooks[n]||x.cssHooks[l],s&&"get"in s&&(a=s.get(e,!0,r)),a===t&&(a=Wt(e,n,i)),"normal"===a&&n in Kt&&(a=Kt[n]),""===r||r?(o=parseFloat(a),r===!0||x.isNumeric(o)?o||0:a):a}}),e.getComputedStyle?(Rt=function(t){return e.getComputedStyle(t,null)},Wt=function(e,n,r){var i,o,a,s=r||Rt(e),l=s?s.getPropertyValue(n)||s[n]:t,u=e.style;return s&&(""!==l||x.contains(e.ownerDocument,e)||(l=x.style(e,n)),Yt.test(l)&&Ut.test(n)&&(i=u.width,o=u.minWidth,a=u.maxWidth,u.minWidth=u.maxWidth=u.width=l,l=s.width,u.width=i,u.minWidth=o,u.maxWidth=a)),l}):a.documentElement.currentStyle&&(Rt=function(e){return e.currentStyle},Wt=function(e,n,r){var i,o,a,s=r||Rt(e),l=s?s[n]:t,u=e.style;return null==l&&u&&u[n]&&(l=u[n]),Yt.test(l)&&!zt.test(n)&&(i=u.left,o=e.runtimeStyle,a=o&&o.left,a&&(o.left=e.currentStyle.left),u.left="fontSize"===n?"1em":l,l=u.pixelLeft+"px",u.left=i,a&&(o.left=a)),""===l?"auto":l});function on(e,t,n){var r=Vt.exec(t);return r?Math.max(0,r[1]-(n||0))+(r[2]||"px"):t}function an(e,t,n,r,i){var o=n===(r?"border":"content")?4:"width"===t?1:0,a=0;for(;4>o;o+=2)"margin"===n&&(a+=x.css(e,n+Zt[o],!0,i)),r?("content"===n&&(a-=x.css(e,"padding"+Zt[o],!0,i)),"margin"!==n&&(a-=x.css(e,"border"+Zt[o]+"Width",!0,i))):(a+=x.css(e,"padding"+Zt[o],!0,i),"padding"!==n&&(a+=x.css(e,"border"+Zt[o]+"Width",!0,i)));return a}function sn(e,t,n){var r=!0,i="width"===t?e.offsetWidth:e.offsetHeight,o=Rt(e),a=x.support.boxSizing&&"border-box"===x.css(e,"boxSizing",!1,o);if(0>=i||null==i){if(i=Wt(e,t,o),(0>i||null==i)&&(i=e.style[t]),Yt.test(i))return i;r=a&&(x.support.boxSizingReliable||i===e.style[t]),i=parseFloat(i)||0}return i+an(e,t,n||(a?"border":"content"),r,o)+"px"}function ln(e){var t=a,n=Gt[e];return n||(n=un(e,t),"none"!==n&&n||(Pt=(Pt||x("