ES-2153 - Added support for multiple files and for updating default p…

…arameters (#426)

* Changed how directory works, now it points to the circleci directory - BREAKING change
Added defaults for directory and schedule

* max 100 files

* Update README.md

---------

Co-authored-by: Lasse Gaardsholt <lasse.gaardsholt@bestseller.com>

.gitignore

@@ -28,3 +28,5 @@ db-secrets
 real-secrets*
 secrets*
 app-secrets
+
+/.idea

README.md

@@ -15,6 +15,7 @@ We have created this as at the time of creation it was nearly impossible to get
 <br/>
 
 ## Getting Started
+
 1. Install the `dependabot-circleci` [GitHub App](https://github.com/apps/dependabot-circleci) in your organization.
 2. You enable `dependabot-circleci` on specific repositories by creating a `dependabot-circleci.yml` configuration file in your repository's `.github` directory. `dependabot-circleci` then raise pull requests to keep the dependencies you configure up-to-date.
 
@@ -37,50 +38,54 @@ reviewers:
   - github_username # for a single user
   - org/team_name # for a whole team (nested teams is the same syntax org/team_name)
 target-branch: main
-directory: "/template" # Used if .github directory is nested inside another directory
+directory: "/.circleci/config.yml" # Folder where the circleci config files are located
 schedule: "monthly" # Options are (daily, weekly, monthly)
 
 ```
 
+dependabot-circleci will recursively scan all the files and folders in the directory specified in the `directory` field for CircleCI config files. If it finds any outdated dependencies, it will raise pull requests against the target branch specified in the `target-branch` field. dependabot-circleci will scan a maximum of 100 entities(folders or yaml/yml files).
+
 ---
 <br/>
 
 ## Configuration options for dependency updates
-The `dependabot-circleci` configuration file, dependabot-circleci.yml, uses YAML syntax. 
-You must store this file in the .github directory of your repository.
 
-| Option                            | Required | Description                            | Default                    |
-| :-------------------------------- | :------: | :------------------------------------- | -------------------------- |
-| [`assignees`](#assignees)         |          | Assignees to set on pull requests      | n/a                        |
-| [`labels`](#labels)               |          | Labels to set on pull requests         | n/a                        |
-| [`reviewers`](#reviewers)         |          | Reviewers to set on pull requests      | n/a                        |
-| [`target-branch`](#target-branch) |          | Branch to create pull requests against | Default branch in the repo |
-| [`directory`](#directory)         |          | Location of .github directory          | Root of repo               |
-| [`schedule`](#schedule)           |          | When to look for updates               | daily                      |
+The `dependabot-circleci` configuration file, dependabot-circleci.yml, uses YAML syntax.
+You must store this file in the .github directory of your repository.
 
+| Option                            | Required | Description                                                                                    | Default                    |
+|:----------------------------------|:--------:|:-----------------------------------------------------------------------------------------------|----------------------------|
+| [`assignees`](#assignees)         |          | Assignees to set on pull requests                                                              | n/a                        |
+| [`labels`](#labels)               |          | Labels to set on pull requests                                                                 | n/a                        |
+| [`reviewers`](#reviewers)         |          | Reviewers to set on pull requests                                                              | n/a                        |
+| [`target-branch`](#target-branch) |          | Branch to create pull requests against                                                         | Default branch in the repo |
+| [`directory`](#directory)         |          | Path to the circleci config file, or folder to be scanned                                      | `/.circleci/config.yml`    |
+| [`schedule`](#schedule)           |          | When to look for updates                                                                       | daily                      |
 
 ---
 <br/>
 
 ## Contributing
+
 We are open for issues, pull requests etc.
 
 ## Running locally
+
 1. Clone the repository
 2. Make sure to have your secrets file in place  
-    2.1 BESTSELLER folks can use Harpocrates to get them from Vault.
+   2.1 BESTSELLER folks can use Harpocrates to get them from Vault.
       ```bash
       harpocrates -f secrets-local.yaml --vault-token $(vault token create -format=json | jq -r '.auth.client_token')
       ```  
-    2.2 Others will have to fill out this template in any other way.  
+   2.2 Others will have to fill out this template in any other way.
       ```json
       {
         "datadog": {
           "api_key": ""
         },
         "github": {
           "app": {
-            "integration_id": ,
+            "integration_id": "",
             "private_key": "",
             "webhook_secret": ""
           },
@@ -103,7 +108,17 @@ We are open for issues, pull requests etc.
       }
       ```
 3. Run `dependabot-circleci` by using Docker compose
-    > `--build` will ensure that the latest version of the code is used
+   > `--build` will ensure that the latest version of the code is used
     ```bash
     docker-compose up --build
-    ```
+    ```
+4. Test worker by sending a POST request to `http://localhost:3000/worker` with the following payload
+    ```bash
+   curl --request POST \
+   --url http://localhost:3000/start \
+   --header 'Content-Type: application/json' \
+   --data '{"Org":"BESTSELLER","Repos": ["dependabot-circleci"]}'
+   ```
+5. If you want to debug the worker without docker: 
+   1. Add the env vars from the docker-compose file to your local environment to match the worker
+   2. Run/Debug in your IDE with the `-worker` flag

api/config_check.go

@@ -59,7 +59,7 @@ func (h *ConfigCheckHandler) Handle(ctx context.Context, eventType, deliveryID s
 	}
 
 	// get content
-	content, _, err := gh.GetRepoContent(ctx, client, Githubinfo.Owner, Githubinfo.RepoName, ".github/dependabot-circleci.yml", commitSHA)
+	content, _, err := gh.GetRepoFileBytes(ctx, client, Githubinfo.Owner, Githubinfo.RepoName, ".github/dependabot-circleci.yml", commitSHA)
 	if err != nil {
 		log.Debug().Err(err).Msg("could not read content of repository")
 		return nil // we dont care

api/controller.go

@@ -28,7 +28,7 @@ type WorkerPayload struct {
 
 var wg sync.WaitGroup
 
-func controllerHandler(w http.ResponseWriter, r *http.Request) {
+func controllerHandler(w http.ResponseWriter, _ *http.Request) {
 	log.Debug().Msg("controllerHandler called")
 
 	orgs, err := pullRepos()
@@ -80,14 +80,16 @@ func shouldRun(schedule string) bool {
 	// check if an update should be run
 	t := time.Now()
 	schedule = strings.ToLower(schedule)
-	if schedule == "monthly" {
-		return (t.Day() == 1)
-	} else if schedule == "weekly" {
-		return (t.Weekday() == 1)
-	} else if schedule == "daily" || schedule == "" {
+	switch schedule {
+	case "daily", "":
 		return true
+	case "weekly":
+		return t.Weekday() == 1
+	case "monthly":
+		return t.Day() == 1
+	default:
+		return false
 	}
-	return false
 }
 
 // PostJSON posts the structs as json to the specified url

circleci/docker.go

@@ -13,56 +13,71 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
-func extractImages(images []*yaml.Node) map[string]*yaml.Node {
+func extractImages(images []*yaml.Node, parameters *map[string]*yaml.Node) map[string]*yaml.Node {
 	updates := map[string]*yaml.Node{}
 	for i := 0; i < len(images); i++ {
 		image := images[i]
 		if image.Value == "image" {
 			image = images[i+1]
 
 			log.Debug().Msg(fmt.Sprintf("current image version: %s", image.Value))
-			imageVersion := findNewestDockerVersion(image.Value)
-			log.Debug().Msg(fmt.Sprintf("new image version: %s", imageVersion))
+			imageName, currentTag, newestTag := findNewestDockerVersion(image.Value, parameters)
+			log.Debug().Msg(fmt.Sprintf("new image version: %s:%s", imageName, newestTag))
 
-			if image.Value != imageVersion {
+			if currentTag != newestTag {
 				oldVersion := image.Value
-				image.Value = imageVersion
+				image.Value = newestTag
 				updates[oldVersion] = image
 			}
 		}
-		baah := extractImages(image.Content)
+		baah := extractImages(image.Content, parameters)
 		for k, v := range baah {
 			updates[k] = v
 		}
 	}
 	return updates
 }
 
-func findNewestDockerVersion(currentVersion string) string {
+func findNewestDockerVersion(currentVersion string, parameters *map[string]*yaml.Node) (imageName, currentTag, newestTag string) {
 	current := strings.Split(currentVersion, ":")
 
 	// check if image has no version tag
 	if len(current) == 1 {
-		return currentVersion
+		return currentVersion, "", ""
 	}
 
 	// check if tag is latest
 	if strings.ToLower(current[1]) == "latest" {
-		return currentVersion
+		return current[0], current[1], current[1]
+	}
+	imageName = current[0]
+	currentTag = current[1]
+
+	if newVersion, hit := cache[currentVersion]; hit {
+		log.Debug().Msgf("Using cached version for image: %s", currentVersion)
+		return imageName, currentTag, newVersion
+	}
+
+	if param := ExtractParameterName(currentVersion); len(param) > 0 {
+		paramDefault, found := (*parameters)[param]
+		if !found {
+			log.Debug().Msgf("Parameter %s not found in parameters", param)
+			return imageName, currentTag, currentTag
+		}
+		currentTag = paramDefault.Value
 	}
 
 	// fix this shit
-	tags, err := getTags(currentVersion)
+	tags, err := getTags(imageName)
 	if err != nil {
 		log.Debug().Err(err)
-		return currentVersion
+		return imageName, currentTag, currentTag
 	}
 
-	versionParts := splitVersion(current[1])
+	versionParts := splitVersion(currentTag)
 	if len(versionParts) == 0 {
-		return currentVersion
+		return imageName, currentTag, currentTag
 	}
-
 	var newTagsList []string
 	for _, tag := range tags {
 		aa := splitVersion(tag)
@@ -95,10 +110,12 @@ func findNewestDockerVersion(currentVersion string) string {
 
 	currentv, _ := version.NewVersion(versionParts["version"])
 	if currentv.GreaterThan(newest) {
-		return currentVersion
+		cache[currentVersion] = currentTag
+		return imageName, currentTag, currentTag
 	}
-
-	return fmt.Sprintf("%s:%s", current[0], newest.Original())
+	newVersion := newest.Original()
+	cache[currentVersion] = newVersion
+	return imageName, currentTag, newVersion
 }
 
 func getTags(circleciTag string) ([]string, error) {
@@ -152,9 +169,9 @@ func splitVersion(version string) map[string]string {
 	}
 
 	matches := myExp.SubexpNames()
-	for i, name := range matches {
-		if i != 0 && name != "" && match[i] != "" {
-			result[name] = match[i]
+	for i, imgName := range matches {
+		if i != 0 && imgName != "" && match[i] != "" {
+			result[imgName] = match[i]
 		}
 	}
 

circleci/orb.go

@@ -12,31 +12,44 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
-func extractOrbs(orbs []*yaml.Node) map[string]*yaml.Node {
+func extractOrbs(orbs []*yaml.Node, parameters *map[string]*yaml.Node) map[string]*yaml.Node {
 	updates := map[string]*yaml.Node{}
 	for i := 0; i < len(orbs); i = i + 2 {
 		orb := orbs[i+1]
 
 		log.Debug().Msg(fmt.Sprintf("current orb version: %s", orb.Value))
-		orbVersion := findNewestOrbVersion(orb.Value)
-		log.Debug().Msg(fmt.Sprintf("new orb version: %s", orbVersion))
+		orbRoot, currentVer, newestVer := findNewestOrbVersion(orb.Value, parameters)
+		log.Debug().Msg(fmt.Sprintf("new orb version: %s@%s", orbRoot, newestVer))
 
-		if orb.Value != orbVersion {
+		if currentVer != newestVer {
 			oldVersion := orb.Value
-			orb.Value = orbVersion
+			orb.Value = newestVer
 			updates[oldVersion] = orb
 		}
 	}
 	return updates
 }
 
-func findNewestOrbVersion(orb string) string {
-
-	orbSplitString := strings.Split(orb, "@")
-
+func findNewestOrbVersion(currentVersion string, parameters *map[string]*yaml.Node) (orbName, currentTag, newestTag string) {
+	orbSplitString := strings.Split(currentVersion, "@")
 	// check if orb is always updated
 	if orbSplitString[1] == "volatile" || strings.HasPrefix(orbSplitString[1], "dev:") {
-		return orbSplitString[1]
+		return orbSplitString[0], orbSplitString[1], orbSplitString[1]
+	}
+	orbName = orbSplitString[0]
+	currentTag = orbSplitString[1]
+
+	if newestTag, hit := cache[currentVersion]; hit {
+		log.Debug().Msgf("Using cached version for orb: %s - %s", orbName, newestTag)
+		return orbName, currentTag, newestTag
+	}
+	if param := ExtractParameterName(currentVersion); len(param) > 0 {
+		paramDefault, found := (*parameters)[param]
+		if !found {
+			log.Debug().Msgf("Parameter %s not found in parameters", param)
+			return orbName, currentTag, currentTag
+		}
+		currentTag = paramDefault.Value
 	}
 
 	CCIApiToken := ""
@@ -48,11 +61,17 @@ func findNewestOrbVersion(orb string) string {
 	client := graphql.NewClient(http.DefaultClient, "https://circleci.com/", "graphql-unstable", CCIApiToken, false)
 
 	// if requests fails, return current version
-	orbInfo, err := api.OrbInfo(client, orbSplitString[0])
+	orbInfo, err := api.OrbInfo(client, orbName)
 	if err != nil {
 		log.Error().Err(err).Msgf("error finding latests orb version failed for orb: %s", orbSplitString[0])
-		return fmt.Sprintf("%s@%s", orbSplitString[0], orbSplitString[1])
+		return orbName, currentTag, currentTag
+	}
+
+	if len(orbInfo.Orb.HighestVersion) == 0 {
+		cache[currentVersion] = currentTag
+		return orbName, currentTag, currentTag
 	}
 
-	return fmt.Sprintf("%s@%s", orbSplitString[0], orbInfo.Orb.HighestVersion)
+	cache[currentVersion] = orbInfo.Orb.HighestVersion
+	return orbName, currentTag, orbInfo.Orb.HighestVersion
 }