Merge pull request #41 from vsoch/add/github-updater

Adding GitHub updater (for releases)

docs/docs/user-guide/user-guide.md

@@ -45,9 +45,70 @@ The following commands are available.
 ?> $ uptodate dockerfile
 
 This command will read one or more Dockerfiles, and tell us if the digest is
-up to date. When run by default, it will automatically update digests.
+up to date. For this parser, an update means the digest of a `FROM` statement, or a build argument
+that follows a particular convention. 
 
-For example, to update a single Dockerfile, you would do:
+#### Digest Update
+
+For digests, you might see that:
+
+```dockerfile
+FROM ubuntu:20.04
+```
+
+is updated to
+
+```dockerfile
+FROM ubuntu:18.04@sha256:9bc830af2bef73276515a29aa896eedfa7bdf4bdbc5c1063b4c457a4bbb8cd79
+```
+
+And then subsequent updates will compare the digest to any known, newer one.
+
+#### Build Arguments
+
+For build arguments, it can only work given that you name them according to
+a known uptodate updater, and you provide a default. Here is a generate format:
+
+```dockerfile
+ARG uptodate_<build-arg-type>_<build-arg-value>=<default>
+```
+
+##### Spack Build Argument
+
+For example, here is how we might specify a build arg for a version of the spack package, "ace"
+
+```dockerfile
+ARG uptodate_spack_ace=6.5.6
+```
+After the updater runs, if it finds a new version 6.5.12, the line will read:
+
+```dockerfile
+ARG uptodate_spack_ace=6.5.12
+```
+
+##### Repository Build Argument
+
+Let's say we are installing spack itself, and we want our updater to find new releases
+on GitHub. The line might look like this:
+
+```
+ARG uptodate_github_spack__spack=v0.16.1
+```
+
+And the double underscore indicates indicates the separation between the organization
+and repository names. So the above will look for new releases on [https://github.com/spack/spack]
+and update as follows:
+
+```
+ARG uptodate_github_spack__spack=v0.16.2
+```
+
+Since we don't see any use cases for a container identifier as an ARG, we don't currently support this.
+But if you do, please [open an issue](https://github.com/vsoch/uptodate/issues).
+
+#### Example
+
+As an example example, to update a single Dockerfile, you would do:
 
 ```bash
 $ ./uptodate dockerfile /path/to/Dockerfile

parsers/docker/build.go

@@ -46,7 +46,7 @@ func parseBuildArg(key string, buildarg config.BuildArg) []parsers.BuildVariable
 	return vars
 }
 
-// parseContainerBuildArg parses a spack build arg
+// parseContainerBuildArg parses a container build arg
 func parseContainerBuildArg(key string, buildarg config.BuildArg) []parsers.BuildVariable {
 
 	// We will return a list of BuildVariable
@@ -80,15 +80,7 @@ func parseContainerBuildArg(key string, buildarg config.BuildArg) []parsers.Buil
 func parseSpackBuildArg(key string, buildarg config.BuildArg) []parsers.BuildVariable {
 
 	// Get versions for current spack package
-	packageUrl := "https://spack.github.io/packages/data/packages/" + buildarg.Name + ".json"
-	response := utils.GetRequest(packageUrl)
-
-	// The response gets parsed into a spack package
-	pkg := spack.SpackPackage{}
-	err := json.Unmarshal([]byte(response), &pkg)
-	if err != nil {
-		log.Fatalf("Issue unmarshalling %s\n", packageUrl)
-	}
+	pkg := spack.GetSpackPackage(buildarg.Name)
 
 	// Get versions based on user preferences
 	versions := pkg.GetVersions(buildarg.Filter, buildarg.StartAt, buildarg.EndAt, buildarg.Skips, buildarg.Includes)

parsers/docker/docker.go

@@ -9,6 +9,8 @@ import (
 
 	lookout "github.com/alecbcs/lookout/update"
 	"github.com/vsoch/uptodate/parsers"
+	"github.com/vsoch/uptodate/parsers/github"
+	"github.com/vsoch/uptodate/parsers/spack"
 	"github.com/vsoch/uptodate/utils"
 )
 
@@ -18,7 +20,7 @@ func GetVersions(container string, filters []string, startAtVersion string, endA
 
 	// Get tags for current container image
 	tagsUrl := "https://crane.ggcr.dev/ls/" + container
-	response := utils.GetRequest(tagsUrl)
+	response := utils.GetRequest(tagsUrl, map[string]string{})
 	tags := strings.Split(response, "\n")
 	sort.Sort(sort.StringSlice(tags))
 
@@ -98,3 +100,105 @@ func UpdateFrom(fromValue []string) Update {
 	}
 	return update
 }
+
+// UpdateArg updates a build arg that is a known pattern
+func UpdateArg(values []string) Update {
+
+	// We will return an update, empty if none
+	update := Update{}
+
+	// This is the full argument with =
+	arg := values[0]
+
+	// If we don't have an = (no default) we cannot update
+	if !strings.Contains(arg, "=") {
+		fmt.Printf("Cannot update %s, does not have a default\n", arg)
+		return update
+	}
+
+	// Keep the original for later comparison
+	original := strings.Join(values, " ")
+
+	// Split into buildarg name and value
+	parts := strings.SplitN(arg, "=", 2)
+
+	name := parts[0]
+	value := parts[1]
+
+	// We can't have empty value
+	if value == "" {
+		fmt.Printf("Cannot update %s, does not have a value\n", arg)
+		return update
+	}
+
+	// Determine if it matches spack or Github
+	if strings.HasPrefix(name, "uptodate_spack") {
+		fmt.Printf("Found spack build arg prefix %s\n", arg)
+
+		name = strings.Replace(name, "uptodate_spack_", "", 1)
+
+		// Get versions for current spack package
+		pkg := spack.GetSpackPackage(name)
+
+		// Should be sorted with newest first
+		if len(pkg.Versions) > 0 {
+
+			updated := parts[0] + "=" + pkg.Versions[0].Name
+
+			// Add any comments back
+			for _, extra := range values[1:] {
+				updated += " " + extra
+			}
+
+			// If the updated version is different from the original, update
+			if updated != original {
+				update = Update{Original: original, Updated: updated}
+
+			} else {
+				fmt.Println("No difference between:", updated, original)
+			}
+		}
+
+	} else if strings.HasPrefix(name, "uptodate_github") {
+
+		fmt.Printf("Found github release build arg prefix %s\n", arg)
+		name = strings.Replace(name, "uptodate_github_", "", 1)
+
+		// The repository name must be separated by __
+		if !strings.Contains(name, "__") {
+			fmt.Printf("Cannot find double underscore to separate org from repo name: %s", name)
+			return update
+		}
+		orgRepo := strings.SplitN(name, "__", 2)
+
+		// Organization __ Repository
+		if orgRepo[0] == "" || orgRepo[1] == "" {
+			fmt.Printf("Org (%s) or repository (%s) is empty, cannot parse.", orgRepo[0], orgRepo[1])
+			return update
+		}
+		repository := orgRepo[0] + "/" + orgRepo[1]
+		fmt.Println(repository)
+		releases := github.GetReleases(repository)
+
+		// The first in the list is the newest release
+		if len(releases) == 0 {
+			fmt.Printf("%s has no releases, cannot update.", repository)
+		}
+		release := releases[0]
+
+		updated := parts[0] + "=" + release.Name
+
+		// Add original content back
+		for _, extra := range values[1:] {
+			updated += " " + extra
+		}
+
+		// If the updated version is different from the original, update
+		if updated != original {
+			update = Update{Original: original, Updated: updated}
+		} else {
+			fmt.Println("No difference between:", updated, original)
+		}
+	}
+	return update
+}

parsers/docker/dockerfile.go

@@ -128,7 +128,7 @@ func (d *Dockerfile) UpdateFroms() {
 		// An "empty" update will be returned if nothing to do
 		newUpdate := UpdateFrom(from.Value)
 		if !reflect.DeepEqual(newUpdate, Update{}) {
-
+			newUpdate.Updated = "FROM " + newUpdate.Updated
 			newUpdate.Original = from.Original
 			newUpdate.LineNo = from.StartIndex()
 			d.Updates = append(d.Updates, newUpdate)
@@ -137,6 +137,23 @@ func (d *Dockerfile) UpdateFroms() {
 	}
 }
 
+// UpdateArgs, updates build args that match a known pattern
+// ARG uptodate_spack_ace=6.5.12  (spack example)
+// ARG uptodate_github_spack__spack=v0.16.1 (github release example)
+func (d *Dockerfile) UpdateArgs() {
+
+	// d.Updates should already be created from Update Froms
+	for _, buildarg := range d.Cmds["arg"] {
+		newUpdate := UpdateArg(buildarg.Value)
+		if !reflect.DeepEqual(newUpdate, Update{}) {
+			newUpdate.Updated = "ARG " + newUpdate.Updated
+			newUpdate.Original = buildarg.Original
+			newUpdate.LineNo = buildarg.StartIndex()
+			d.Updates = append(d.Updates, newUpdate)
+		}
+	}
+}
+
 // ReplaceFroms simply replaces found FROM with a known value
 // This is typically run instead of UpdateFroms
 func (d *Dockerfile) ReplaceFroms(name string, tag string) {
@@ -198,7 +215,7 @@ func (d *Dockerfile) Write() {
 		fmt.Printf("Updating %s to %s\n", update.Original, update.Updated)
 
 		// This ensures we keep the tag preserved for future checks, but change the file so it rebuilds
-		lines[update.LineNo] = "FROM " + update.Updated
+		lines[update.LineNo] = update.Updated
 	}
 	content := strings.Join(lines, "\n")
 	utils.WriteFile(d.Path, content)
@@ -240,6 +257,7 @@ func (s *DockerfileParser) AddDockerfile(root string, path string) {
 	dockerfile := Dockerfile{Path: path, Root: root}
 	dockerfile.ParseCommands()
 	dockerfile.UpdateFroms()
+	dockerfile.UpdateArgs()
 	s.Dockerfiles = append(s.Dockerfiles, dockerfile)
 }
 

parsers/github/github.go

@@ -0,0 +1,98 @@
+package github
+
+import (
+	"encoding/json"
+	"github.com/vsoch/uptodate/utils"
+	"log"
+	"time"
+)
+
+type Releases []struct {
+	URL             string    `json:"url"`
+	HTMLURL         string    `json:"html_url"`
+	AssetsURL       string    `json:"assets_url"`
+	UploadURL       string    `json:"upload_url"`
+	TarballURL      string    `json:"tarball_url"`
+	ZipballURL      string    `json:"zipball_url"`
+	ID              int       `json:"id"`
+	NodeID          string    `json:"node_id"`
+	TagName         string    `json:"tag_name"`
+	TargetCommitish string    `json:"target_commitish"`
+	Name            string    `json:"name"`
+	Body            string    `json:"body"`
+	Draft           bool      `json:"draft"`
+	Prerelease      bool      `json:"prerelease"`
+	CreatedAt       time.Time `json:"created_at"`
+	PublishedAt     time.Time `json:"published_at"`
+	Author          struct {
+		Login             string `json:"login"`
+		ID                int    `json:"id"`
+		NodeID            string `json:"node_id"`
+		AvatarURL         string `json:"avatar_url"`
+		GravatarID        string `json:"gravatar_id"`
+		URL               string `json:"url"`
+		HTMLURL           string `json:"html_url"`
+		FollowersURL      string `json:"followers_url"`
+		FollowingURL      string `json:"following_url"`
+		GistsURL          string `json:"gists_url"`
+		StarredURL        string `json:"starred_url"`
+		SubscriptionsURL  string `json:"subscriptions_url"`
+		OrganizationsURL  string `json:"organizations_url"`
+		ReposURL          string `json:"repos_url"`
+		EventsURL         string `json:"events_url"`
+		ReceivedEventsURL string `json:"received_events_url"`
+		Type              string `json:"type"`
+		SiteAdmin         bool   `json:"site_admin"`
+	} `json:"author"`
+	Assets []struct {
+		URL                string    `json:"url"`
+		BrowserDownloadURL string    `json:"browser_download_url"`
+		ID                 int       `json:"id"`
+		NodeID             string    `json:"node_id"`
+		Name               string    `json:"name"`
+		Label              string    `json:"label"`
+		State              string    `json:"state"`
+		ContentType        string    `json:"content_type"`
+		Size               int       `json:"size"`
+		DownloadCount      int       `json:"download_count"`
+		CreatedAt          time.Time `json:"created_at"`
+		UpdatedAt          time.Time `json:"updated_at"`
+		Uploader           struct {
+			Login             string `json:"login"`
+			ID                int    `json:"id"`
+			NodeID            string `json:"node_id"`
+			AvatarURL         string `json:"avatar_url"`
+			GravatarID        string `json:"gravatar_id"`
+			URL               string `json:"url"`
+			HTMLURL           string `json:"html_url"`
+			FollowersURL      string `json:"followers_url"`
+			FollowingURL      string `json:"following_url"`
+			GistsURL          string `json:"gists_url"`
+			StarredURL        string `json:"starred_url"`
+			SubscriptionsURL  string `json:"subscriptions_url"`
+			OrganizationsURL  string `json:"organizations_url"`
+			ReposURL          string `json:"repos_url"`
+			EventsURL         string `json:"events_url"`
+			ReceivedEventsURL string `json:"received_events_url"`
+			Type              string `json:"type"`
+			SiteAdmin         bool   `json:"site_admin"`
+		} `json:"uploader"`
+	} `json:"assets"`
+}
+
+func GetReleases(name string) Releases {
+
+	url := "https://api.github.com/repos/" + name + "/releases"
+
+	headers := make(map[string]string)
+	headers["Accept"] = "application/vnd.github.v3+json"
+	response := utils.GetRequest(url, headers)
+
+	// The response gets parsed into a spack package
+	releases := Releases{}
+	err := json.Unmarshal([]byte(response), &releases)
+	if err != nil {
+		log.Fatalf("Issue unmarshalling releases data structure\n")
+	}
+	return releases
+}