Add Integration tests

Makefile

@@ -20,6 +20,9 @@ CONTROL_NAMESPACE  := default
 CONTROL_KUBECONFIG := dev/target-kubeconfig.yaml
 TARGET_KUBECONFIG  := dev/target-kubeconfig.yaml
 
+LEADER_ELECT 	   := "true"
+MACHINE_SAFETY_OVERSHOOTING_PERIOD:=1m
+
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
 GOBIN=$(shell go env GOPATH)/bin
@@ -48,7 +51,8 @@ start:
 			--machine-safety-apiserver-statuscheck-timeout=30s \
 			--machine-safety-apiserver-statuscheck-period=1m \
 			--machine-safety-orphan-vms-period=30m \
-			--machine-safety-overshooting-period=1m \
+			--machine-safety-overshooting-period=$(MACHINE_SAFETY_OVERSHOOTING_PERIOD) \
+			--leader-elect=$(LEADER_ELECT) \
 			--v=3
 
 #################################################################

docs/development/cp_support_new.md

@@ -1,16 +1,16 @@
 # Adding support for a new provider
 
-Steps to be followed while implementing a new (hyperscale) provider are mentioned below. This is the easiest way to add a new provider support using a blueprint code.
+Steps to be followed while implementing a new (hyperscale) provider are mentioned below. This is the easiest way to add new provider support using a blueprint code.
 
-However, you may also develop your machine controller from scratch which would provide you more flexibility. However make sure that your custom machine controller adhere's to the `Machine.Status` struct defined in the [MachineAPIs](/pkg/apis/machine/types.go) to make sure the MCM is able to act with higher level controllers like MachineSet and MachineDeployment controller. The key is the `Machine.Status.CurrentStatus.Phase` key that indicates the status of the machine object.
+However, you may also develop your machine controller from scratch, which would provide you with more flexibility. First, however, make sure that your custom machine controller adheres to the `Machine.Status` struct defined in the [MachineAPIs](/pkg/apis/machine/types.go). This will make sure the MCM can act with higher-level controllers like MachineSet and MachineDeployment controller. The key is the `Machine.Status.CurrentStatus.Phase` key that indicates the status of the machine object.
 
-Our strong recommendation would be to follow the steps below as this provides most flexibility required to support machine management for adding new providers. And if you feel to extend the functionality feel free to update our [machine controller libraries](/pkg/util/provider).
+Our strong recommendation would be to follow the steps below. This provides the most flexibility required to support machine management for adding new providers. And if you feel to extend the functionality, feel free to update our [machine controller libraries](/pkg/util/provider).
 
 ## Setting up your repository
 
-1. Create a new empty repository named `machine-controller-manager-provider-{provider-name}` on github username/project. Do not initialize this repository with a README.
-1. Copy the remote repository `URL` (HTTPS/SSH) to this repository which is displayed once you create this repository.
-1. Now on your local system, create directories as required. {your-github-username} given below could also be {github-project} depending on where you have created the new repository.
+1. Create a new empty repository named `machine-controller-manager-provider-{provider-name}` on GitHub username/project. Do not initialize this repository with a README.
+1. Copy the remote repository `URL` (HTTPS/SSH) to this repository displayed once you create this repository.
+1. Now, on your local system, create directories as required. {your-github-username} given below could also be {github-project} depending on where you have created the new repository.
     ```bash
     mkdir -p $GOPATH/src/github.com/{your-github-username}
     ```
@@ -26,22 +26,22 @@ Our strong recommendation would be to follow the steps below as this provides mo
     ```bash
     mv machine-controller-manager-provider-sampleprovider machine-controller-manager-provider-{provider-name}
     ```
-1. Navigate into the newly created directory.
+1. Navigate into the newly-created directory.
     ```bash
     cd machine-controller-manager-provider-{provider-name}
     ```
 1. Update the remote `origin` URL to the newly created repository's URL you had copied above.
     ```bash
     git remote set-url origin git@github.com:{your-github-username}/machine-controller-manager-provider-{provider-name}.git
     ```
-1. Rename github project from `gardener` to `{github-org/your-github-username}` wherever you have cloned the repository above. Also edit all occurrences of the word `sampleprovider` to `{provider-name}` in the code. Use the hack script given below to do the same.
+1. Rename GitHub project from `gardener` to `{github-org/your-github-username}` wherever you have cloned the repository above. Also, edit all occurrences of the word `sampleprovider` to `{provider-name}` in the code. Then, use the hack script given below to do the same.
     ```bash
     make rename-project PROJECT_NAME={github-org/your-github-username} PROVIDER_NAME={provider-name}
     eg:
         make rename-project PROJECT_NAME=gardener PROVIDER_NAME=AmazonWebServices (or)
         make rename-project PROJECT_NAME=githubusername PROVIDER_NAME=AWS
     ```
-1. Now commit your changes and push it upstream.
+1. Now, commit your changes and push them upstream.
     ```bash
     git add -A
     git commit -m "Renamed SampleProvide to {provider-name}"
@@ -50,36 +50,36 @@ Our strong recommendation would be to follow the steps below as this provides mo
 
 ## Code changes required
 
-The contract between he Machine Controller Manager (MCM) and the Machine Controller (MC) AKA driver has been [documented here](machine_error_codes.md) and the [machine error codes can be found here](/pkg/util/provider/machinecodes/codes/codes.go). You may refer to them for any queries.
+The contract between the Machine Controller Manager (MCM) and the Machine Controller (MC) AKA driver has been [documented here](machine_error_codes.MD) and the [machine error codes can be found here](/pkg/util/provider/machinecodes/codes/codes.go). You may refer to them for any queries.
 
 :warning:
-- Keep in mind that, **there should to be a unique way to map between machine objects and VMs**. This can be done by mapping machine object names with VM-Name/ tags/ other metadata.
-- Optionally there should also be a unique way to map a VM to it's machine class object. This can be done by tagging VM objects with tags/resource-groups associated with the machine class.
+- Keep in mind that **there should be a unique way to map between machine objects and VMs**. This can be done by mapping machine object names with VM-Name/ tags/ other metadata.
+- Optionally, there should also be a unique way to map a VM to its machine class object. This can be done by tagging VM objects with tags/resource groups associated with the machine class.
 
 #### Steps to integrate
 
 1. Update the `pkg/provider/apis/provider_spec.go` specification file to reflect the structure of the `ProviderSpec` blob. It typically contains the machine template details in the `MachineClass` object. Follow the sample spec provided already in the file. A sample provider specification can be found [here](https://github.com/gardener/machine-controller-manager-provider-aws/blob/master/pkg/aws/apis/aws_provider_spec.go).
 1. Fill in the methods described at `pkg/provider/core.go` to manage VMs on your cloud provider. Comments are provided above each method to help you fill them up with desired `REQUEST` and `RESPONSE` parameters.
     - A sample provider implementation for these methods can be found [here](https://github.com/gardener/machine-controller-manager-provider-aws/blob/master/pkg/aws/core.go).
     - Fill in the required methods `CreateMachine()`, and `DeleteMachine()` methods.
-    - Optionally fill in methods like `GetMachineStatus()`, `ListMachines()`, and `GetVolumeIDs()`. You may choose to fill these, once the working of the required methods seem to be working.
-        - `GetVolumeIDs()` expects VolumeIDs to be decoded from the volumeSpec based on the cloud provider.
-    - There is also an OPTIONAL method `GenerateMachineClassForMigration()` that helps in migration of `{ProviderSpecific}MachineClass` to `MachineClass` CR (custom resource). This only makes sense if you have an existing implementation (in-tree) acting on different CRD types and you would like to migrate this. If not you MUST return an error (machine error UNIMPLEMENTED) to avoid processing this step.
-1. Perform validation of APIs that you have described and make it a part of your methods as required at each requests.
+    - Optionally fill in methods like `GetMachineStatus()`, `ListMachines()`, and `GetVolumeIDs()`. You may choose to fill these once the working of the required methods seems to be working.
+    - `GetVolumeIDs()` expects VolumeIDs to be decoded from the volumeSpec based on the cloud provider.
+    - There is also an OPTIONAL method `GenerateMachineClassForMigration()` that helps in migration of `{ProviderSpecific}MachineClass` to `MachineClass` CR (custom resource). This only makes sense if you have an existing implementation (in-tree) acting on different CRD types. You would like to migrate this. If not, you MUST return an error (machine error UNIMPLEMENTED) to avoid processing this step.
+1. Perform validation of APIs that you have described and make it a part of your methods as required at each request.
 1. Write unit tests to make it work with your implementation by running `make test`.
     ```bash
     make test
     ```
-1. Re-generate the vendors, to update any new vendors imported.
+1. Re-generate the vendors to update any new vendors imported.
     ```bash
     make revendor
     ```
-1. Update the sample YAML files on `kubernetes/` directory to provide sample files through which the working of the machine controller can be tested.
+1. Update the sample YAML files on the `kubernetes/` directory to provide sample files through which the working of the machine controller can be tested.
 1. Update `README.md` to reflect any additional changes
 
 ## Testing your code changes
 
-Make sure `$TARGET_KUBECONFIG` points to the cluster where you wish to manage machines. `$CONTROL_NAMESPACE` represents the namespaces where MCM is looking for machine CR objects, and `$CONTROL_KUBECONFIG` points to the cluster which holds these machine CRs.
+Make sure `$TARGET_KUBECONFIG` points to the cluster where you wish to manage machines. Likewise, `$CONTROL_NAMESPACE` represents the namespaces where MCM is looking for machine CR objects, and `$CONTROL_KUBECONFIG` points to the cluster that holds these machine CRs.
 
 1. On the first terminal running at `$GOPATH/src/github.com/{github-org/your-github-username}/machine-controller-manager-provider-{provider-name}`,
     - Run the machine controller (driver) using the command below.
@@ -91,7 +91,7 @@ Make sure `$TARGET_KUBECONFIG` points to the cluster where you wish to manage ma
         ```bash
         git clone git@github.com:gardener/machine-controller-manager.git
         ```
-    - Navigate to the newly created directory.
+    - Navigate to the newly-created directory.
         ```bash
         cd machine-controller-manager
         ```
@@ -117,12 +117,12 @@ Make sure `$TARGET_KUBECONFIG` points to the cluster where you wish to manage ma
         ```bash
         kubectl apply -f kubernetes/machine.yaml
         ```
-    - Once machine joins, you can test by deploying a machine-deployment.
+    - Once the machine joins, you can test by deploying a machine-deployment.
     - Deploy the `machine-deployment` object and make sure it joins the cluster successfully.
         ```bash
         kubectl apply -f kubernetes/machine-deployment.yaml
         ```
-    - Make sure to delete both the `machine` and `machine-deployment` object after use.
+    - Make sure to delete both the `machine` and `machine-deployment` objects after use.
         ```bash
         kubectl delete -f kubernetes/machine.yaml
         kubectl delete -f kubernetes/machine-deployment.yaml

docs/development/integration_tests.md

@@ -0,0 +1,34 @@
+# Running Integration tests
+
+Integration tests for `machine-controller-manager` and `machine-controller-manager-provider-{provider-name}` can be executed manually by following below steps.
+
+1. Clone the repository `machine-controller-manager-provider-{provider-name}` on your local system.
+1. Navigate to `machine-controller-manager-provider-{provider-name}` directory and then create a `dev` sub-directory in it.
+1. Copy the kubeconfig of the kubernetes cluster from where you wish to manage the machines into `dev/control-kubeconfig.yaml`. 
+1. (optional) Copy the kubeconfig of the kubernetes cluster where you wish to deploy the machines into `dev/target-kubeconfig.yaml`. If you do this, also update the `Makefile` variable TARGET_KUBECONFIG to point to `dev/target-kubeconfig.yaml`.
+1. If the kubernetes cluster referred by `dev/control-kubeconfig.yaml` is a gardener shoot cluster, then
+    - Create a secret that contains the provider secret and cloud-config into the kubernetes cluster.
+    - Create a `dev/machineclassv1.yaml` file. The value of `providerSpec.secretRef.name` should be the secret created in the previous step. The name of the machineclass itself should be `test-mc-v1`. 
+    - (optional) Create an additional `dev/machineclassv2.yaml` file similar to above but with a bigger machine type. If you do this, update the `Makefile` variable MACHINECLASS_V2 to point to `dev/machineclassv2.yaml`. 
+    - If tags for controllers container images are known, update the `Makefile` variables MCM_IMAGE_TAG and MC_IMAGE_TAG accordingly. These will be used along with `kubernetes/deployment.yaml` to deploy controllers into the cluster. If not, the controllers will be started in the local system. `machine-controller-manager` repository will be cloned automatically for this. incase, this repository already exists in local system, then create a softlink as below. It also helps to test changes in `machine-controller-manager` quickly.
+    ```bash
+    ln -sf <path-for-machine-controller-manager-repo> dev/mcm
+    ```
+1. If the cluster referred by `dev/control-kubeconfig.yaml` is a gardener seed cluster and the tags for controllers container images are known, update the `Makefile` variables MCM_IMAGE_TAG and MC_IMAGE_TAG accordingly. These will be used to update the existing controllers running the cluster.
+1. There is a rule `test-integration` in the `Makefile`, which can be used to start the integration test:
+    ```bash
+    $ make test-integration 
+    Starting integration tests...
+    Running Suite: Controller Suite
+    ===============================
+    ```
+1. The controllers log files (mcm_process.log and mc_process.log) are stored as temporary files and can be used later.
+
+## Adding integration tests for new providers
+
+For a new provider, [Running Integration tests](#Running-Integration-tests) works with no changes. But for the orphan resource test cases to work correctly, the provider-specific API calls and the rti should be implemented. Please check `machine-controller-manager-provider-aws` for referrence.
+
+## Extending integration tests
+
+- If the testcases for all providers has to be extended, then [ControllerTests](pkg/test/integration/common/framework.go#L481) should be updated. Common test-cases for machine|machineDeployment creation|deletion|scaling have been packaged into [ControllerTests](pkg/test/integration/common/framework.go#L481). But they can be extended any time.
+- If the test cases for a specific provider has to be extended, then the changes should be done in the `machine-controller-manager-provider-{provider-name}` repository. For example, if the tests for `machine-controller-manager-provider-aws` are to be extended then make changes to `test/integration/controller/controller_test.go` inside the `machine-controller-manager-provider-aws` repository. `commons` contains the Cluster and Clientset objects that makes it easy to extend the tests.

go.mod

@@ -13,6 +13,7 @@ require (
 	github.com/cenkalti/backoff/v4 v4.1.0
 	github.com/davecgh/go-spew v1.1.1
 	github.com/emicklei/go-restful v2.9.6+incompatible // indirect
+	github.com/go-git/go-git/v5 v5.3.0
 	github.com/go-openapi/spec v0.19.2
 	github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7 // indirect
 	github.com/googleapis/gnostic v0.2.0 // indirect
@@ -31,6 +32,7 @@ require (
 	google.golang.org/api v0.4.0
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	k8s.io/api v0.0.0-20190918155943-95b840bb6a1f
+	k8s.io/apiextensions-apiserver v0.0.0-20190918161926-8f644eb6e783
 	k8s.io/apimachinery v0.0.0-20190913080033-27d36303b655
 	k8s.io/apiserver v0.0.0-20190918160949-bfa5e2e684ad
 	k8s.io/client-go v0.0.0-20190918160344-1fbdaa4c8d90
@@ -43,9 +45,11 @@ require (
 )
 
 replace (
+	github.com/go-git/go-git/v5 => github.com/go-git/go-git/v5 v5.3.0
 	github.com/onsi/ginkgo => github.com/onsi/ginkgo v1.8.0
 	github.com/onsi/gomega => github.com/onsi/gomega v1.5.0
 	k8s.io/api => k8s.io/api v0.0.0-20190918155943-95b840bb6a1f // kubernetes-1.16.0
+	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.0.0-20190918161926-8f644eb6e783 // kubernetes-1.16.0
 	k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20190913080033-27d36303b655 // kubernetes-1.16.0
 	k8s.io/apiserver => k8s.io/apiserver v0.0.0-20190918160949-bfa5e2e684ad // kubernetes-1.16.0
 	k8s.io/client-go => k8s.io/client-go v0.0.0-20190918160344-1fbdaa4c8d90 // kubernetes-1.16.0