GitHub reconciliation watcher (#121)

Sync releases and opened PRs every 10 minutes

apis/v1alpha1/github_repository.go

@@ -59,6 +59,9 @@ type GitHubRepositoryStatus struct {
 	// Webhook represents the installed Webhook information for the GitHub
 	// Repository.
 	Webhook *GitHubHook `json:"webhook"`
+
+	// Reconciliation represents the status of the repository reconciliation.
+	Reconciliation GitHubReconciliation `json:"reconciliation"`
 }
 
 // GitHubHook represents the status object for a GiHub Webhook for the CRD.
@@ -80,6 +83,11 @@ type GitHubRelease struct {
 	Deployment  *Deployment `json:"deployment,omitempty"`
 }
 
+// GitHubReconciliation represents the status of the repository reconciliation.
+type GitHubReconciliation struct {
+	LastUpdate *metav1.Time `json:"last_update"`
+}
+
 // Deployment represents a linking between a GitHub deployment and a network
 // policy. Through the release information we can determine a specific domain.
 type Deployment struct {

cmd/heighliner/github_repository.go

@@ -3,11 +3,11 @@ package main
 import (
 	"log"
 	"os"
+	"time"
 
 	"github.com/jelmersnoeck/kubekit"
 	flags "github.com/jessevdk/go-flags"
 	"github.com/manifoldco/heighliner/internal/githubrepository"
-
 	"github.com/spf13/cobra"
 )
 
@@ -20,10 +20,11 @@ var (
 	}
 
 	ghpcFlags struct {
-		Namespace    string `long:"namespace" env:"NAMESPACE" description:"The namespace we'll watch for CRDs. By default we'll watch all namespaces."`
-		Domain       string `long:"domain" env:"DOMAIN" description:"The domain name used for callbacks" required:"true"`
-		InsecureSSL  bool   `long:"insecure-ssl" env:"INSECURE_SSL" description:"Allow insecure callbacks to the webhook"`
-		CallbackPort string `long:"callback-port" env:"CALLBACK_PORT" description:"The port to run the callbacks server on" default:":8080"`
+		Namespace            string `long:"namespace" env:"NAMESPACE" description:"The namespace we'll watch for CRDs. By default we'll watch all namespaces."`
+		Domain               string `long:"domain" env:"DOMAIN" description:"The domain name used for callbacks" required:"true"`
+		InsecureSSL          bool   `long:"insecure-ssl" env:"INSECURE_SSL" description:"Allow insecure callbacks to the webhook"`
+		CallbackPort         string `long:"callback-port" env:"CALLBACK_PORT" description:"The port to run the callbacks server on" default:":8080"`
+		ReconciliationPeriod string `long:"reconciliation-period" env:"RECONCILIATION_PERIOD" description:"How often the controller should check for Github changes missed by webhooks" default:"10m"`
 	}
 )
 
@@ -44,10 +45,18 @@ func ghpcCommand(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	period, err := time.ParseDuration(ghpcFlags.ReconciliationPeriod)
+	if err != nil {
+		log.Printf("Could not parse Reconciation Period duration %s: %s\n",
+			ghpcFlags.ReconciliationPeriod, err)
+		return err
+	}
+
 	cfg := githubrepository.Config{
-		Domain:       ghpcFlags.Domain,
-		InsecureSSL:  ghpcFlags.InsecureSSL,
-		CallbackPort: ghpcFlags.CallbackPort,
+		Domain:               ghpcFlags.Domain,
+		InsecureSSL:          ghpcFlags.InsecureSSL,
+		CallbackPort:         ghpcFlags.CallbackPort,
+		ReconciliationPeriod: period,
 	}
 
 	ctrl, err := githubrepository.NewController(rcfg, cs, ghpcFlags.Namespace, cfg)

docs/design/github-connector.md

@@ -13,6 +13,11 @@ The connector also takes care of setting up a callback server, allowing GitHub
 to send these events to the cluster. Once a new Release or PullRequest is
 detected, it will be stored accordingly to the associated GitHub Repository CRD.
 
+GitHub webhooks are not retried in case of failure, to mitigate any synchronization
+problems, the connector tries to reconciliate its known releases with GitHub's list
+of releases and opened pull requests every 10 minutes. This period can be configured
+with the flag `--reconciliation-period`.
+
 Lastly, the connector also monitors the NetworkPolicies. These policies indicate
 which Microservices have associated releases. If these releases are Preview
 releases, the connector will create Deployment objects and link the generated

go.mod

@@ -1,16 +1,23 @@
 module github.com/manifoldco/heighliner
 
 require (
+	4d63.com/gochecknoglobals v0.0.0-20180908201037-5090db600a84 // indirect
+	4d63.com/gochecknoinits v0.0.0-20180528051558-14d5915061e5 // indirect
 	cloud.google.com/go v0.23.0
 	github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78
 	github.com/Azure/go-autorest v9.9.0+incompatible
 	github.com/MakeNowJust/heredoc v0.0.0-20171113091838-e9091a26100e
 	github.com/PuerkitoBio/purell v1.1.0
 	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578
+	github.com/alecthomas/gocyclo v0.0.0-20150208221726-aa8f8b160214 // indirect
+	github.com/alexflint/go-arg v0.0.0-20180516182405-f7c0423bd11e // indirect
+	github.com/alexflint/go-scalar v0.0.0-20170216020425-e80c3b7ed292 // indirect
+	github.com/alexkohler/nakedret v0.0.0-20171106223215-c0e305a4f690 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20180315120708-ccb8e960c48f
 	github.com/aws/aws-sdk-go v1.13.54
 	github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973
-	github.com/davecgh/go-spew v1.1.0
+	github.com/client9/misspell v0.3.4 // indirect
+	github.com/davecgh/go-spew v1.1.1
 	github.com/dchest/blake2b v1.0.0
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/docker/distribution v0.0.0-20170726174610-edc3ab29cdff
@@ -38,13 +45,15 @@ require (
 	github.com/gogo/protobuf v1.0.0
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
 	github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7
-	github.com/golang/protobuf v1.1.0
+	github.com/golang/lint v0.0.0-20180702182130-06c8688daad7 // indirect
+	github.com/golang/protobuf v1.2.0
 	github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a
 	github.com/google/go-github v15.0.0+incompatible
 	github.com/google/go-querystring v0.0.0-20170111101155-53e6ce116135
 	github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf
 	github.com/googleapis/gnostic v0.2.0
 	github.com/gophercloud/gophercloud v0.0.0-20180515014705-282f25e4025d
+	github.com/gordonklaus/ineffassign v0.0.0-20180909121442-1003c8bd00dc // indirect
 	github.com/gorilla/context v1.1.1
 	github.com/gorilla/mux v1.6.2
 	github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7
@@ -55,17 +64,23 @@ require (
 	github.com/inconshreveable/mousetrap v1.0.0
 	github.com/jelmersnoeck/kubekit v0.0.0-20180605073004-c5b8ec922b30
 	github.com/jessevdk/go-flags v1.4.0
+	github.com/jgautheron/goconst v0.0.0-20170703170152-9740945f5dcb // indirect
 	github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8
 	github.com/json-iterator/go v0.0.0-20180315132816-ca39e5af3ece
 	github.com/juju/ratelimit v1.0.1
+	github.com/kisielk/errcheck v1.1.0 // indirect
 	github.com/mailru/easyjson v0.0.0-20180323154445-8b799c424f57
 	github.com/matttproud/golang_protobuf_extensions v1.0.0
+	github.com/mdempsky/maligned v0.0.0-20180708014732-6e39bd26a8c8 // indirect
+	github.com/mdempsky/unconvert v0.0.0-20180703203632-1a9a0a0a3594 // indirect
+	github.com/mibk/dupl v1.0.0 // indirect
 	github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7
 	github.com/mitchellh/mapstructure v0.0.0-20180511142126-bb74f1db0675
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd
 	github.com/modern-go/reflect2 v0.0.0-20180228065516-1df9eeb2bb81
 	github.com/opencontainers/go-digest v1.0.0-rc1
 	github.com/opencontainers/image-spec v1.0.1
+	github.com/opennota/check v0.0.0-20180911053232-0c771f5545ff // indirect
 	github.com/pborman/uuid v0.0.0-20170612153648-e790cca94e6c
 	github.com/petar/GoLLRB v0.0.0-20130427215148-53be0d36a84c
 	github.com/peterbourgon/diskv v2.0.1+incompatible
@@ -75,23 +90,29 @@ require (
 	github.com/prometheus/common v0.0.0-20180518154759-7600349dcfe1
 	github.com/prometheus/procfs v0.0.0-20180408092902-8b1c2da0d56d
 	github.com/russross/blackfriday v0.0.0-20151117072312-300106c228d5
+	github.com/securego/gosec v0.0.0-20181005111228-e0a150bfa369 // indirect
 	github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95
 	github.com/sirupsen/logrus v1.0.4
 	github.com/spf13/cobra v0.0.0-20180531180338-1e58aa3361fd
 	github.com/spf13/pflag v1.0.1
-	github.com/stretchr/testify v1.2.1
+	github.com/stretchr/testify v1.2.2
+	github.com/stripe/safesql v0.0.0-20171221195208-cddf355596fe // indirect
+	github.com/tsenart/deadcode v0.0.0-20160724212837-210d2dc333e9 // indirect
+	github.com/walle/lll v0.0.0-20160702150637-8b13b3fbf731 // indirect
 	golang.org/x/crypto v0.0.0-20180524125353-159ae71589f3
+	golang.org/x/lint v0.0.0-20180702182130-06c8688daad7 // indirect
 	golang.org/x/net v0.0.0-20180524181706-dfa909b99c79
 	golang.org/x/oauth2 v0.0.0-20180523224158-770e5ebd4ab2
 	golang.org/x/sys v0.0.0-20180524135853-04b83988a018
 	golang.org/x/text v0.3.0
-	golang.org/x/tools v0.0.0-20180522181625-1937f90a1bb4
+	golang.org/x/tools v0.0.0-20181008205924-a2b3f7f249e9
 	google.golang.org/appengine v1.0.0
 	gopkg.in/gcfg.v1 v1.2.3
 	gopkg.in/inf.v0 v0.9.1
 	gopkg.in/mgo.v2 v2.0.0-20160818020120-3f83fa500528
 	gopkg.in/warnings.v0 v0.1.2
 	gopkg.in/yaml.v2 v2.2.1
+	honnef.co/go/tools v0.0.0-20180920025451-e3ad64cb4ed3 // indirect
 	k8s.io/api v0.0.0-20180127130940-acf347b865f2
 	k8s.io/apiextensions-apiserver v0.0.0-20180206093320-f1425805c033
 	k8s.io/apimachinery v0.0.0-20180126010752-19e3f5aa3adc
@@ -102,5 +123,7 @@ require (
 	k8s.io/kube-openapi v0.0.0-20180522000509-67793630244c
 	k8s.io/kubernetes v1.9.8
 	k8s.io/utils v0.0.0-20180208044234-258e2a2fa645
+	mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed // indirect
+	mvdan.cc/unparam v0.0.0-20181009154202-6669894d00e9 // indirect
 	vbom.ml/util v0.0.0-20170409195630-256737ac55c4
 )