Merge pull request #3 from pjdubya/feat/delay-days-updates-from-pr

Rename param to defer-days, add reporting for deferred containers, general simplifications

cmd/root.go

@@ -39,7 +39,7 @@ var (
 	disableContainers []string
 	notifier          t.Notifier
 	timeout           time.Duration
-	delayDays         int
+	deferDays         int
 	lifecycleHooks    bool
 	rollingRestart    bool
 	scope             string
@@ -97,7 +97,7 @@ func PreRun(cmd *cobra.Command, _ []string) {
 
 	enableLabel, _ = f.GetBool("label-enable")
 	disableContainers, _ = f.GetStringSlice("disable-containers")
-	delayDays, _ = f.GetInt("delay-days")
+	deferDays, _ = f.GetInt("defer-days")
 	lifecycleHooks, _ = f.GetBool("enable-lifecycle-hooks")
 	rollingRestart, _ = f.GetBool("rolling-restart")
 	scope, _ = f.GetString("scope")
@@ -290,9 +290,9 @@ func writeStartupMessage(c *cobra.Command, sched time.Time, filtering string) {
 		until := formatDuration(time.Until(sched))
 		startupLog.Info("Scheduling first run: " + sched.Format("2006-01-02 15:04:05 -0700 MST"))
 		startupLog.Info("Note that the first check will be performed in " + until)
-		delayDays, _ = c.PersistentFlags().GetInt("delay-days")
-		if delayDays > 0 {
-			startupLog.Infof("Container updates will be delayed until %d day(s) after image creation.", delayDays)
+		deferDays, _ = c.PersistentFlags().GetInt("defer-days")
+		if deferDays > 0 {
+			startupLog.Infof("Container updates will be deferred until %d day(s) after image creation.", deferDays)
 		}
 	} else if runOnce, _ := c.PersistentFlags().GetBool("run-once"); runOnce {
 		startupLog.Info("Running a one time update.")
@@ -370,7 +370,7 @@ func runUpdatesWithNotifications(filter t.Filter) *metrics.Metric {
 		NoRestart:       noRestart,
 		Timeout:         timeout,
 		MonitorOnly:     monitorOnly,
-		DelayDays:       delayDays,
+		DeferDays:       deferDays,
 		LifecycleHooks:  lifecycleHooks,
 		RollingRestart:  rollingRestart,
 		LabelPrecedence: labelPrecedence,
@@ -383,9 +383,10 @@ func runUpdatesWithNotifications(filter t.Filter) *metrics.Metric {
 	notifier.SendNotification(result)
 	metricResults := metrics.NewMetric(result)
 	notifications.LocalLog.WithFields(log.Fields{
-		"Scanned": metricResults.Scanned,
-		"Updated": metricResults.Updated,
-		"Failed":  metricResults.Failed,
+		"Scanned":  metricResults.Scanned,
+		"Updated":  metricResults.Updated,
+		"Deferred": metricResults.Deferred,
+		"Failed":   metricResults.Failed,
 	}).Info("Session done")
 	return metricResults
 }

docs/arguments.md

@@ -381,12 +381,12 @@ Environment Variable: WATCHTOWER_HTTP_API_METRICS
              Default: false
 ```
 
-## Delayed Update
+## Deferred Update
 Only update container to latest version of image if some number of days have passed since it has been published. This option may be useful for those who wish to avoid updating prior to the new version having some time in the field prior to updating in case there are critical defects found and released in a subsequent version.
 
 ```text
-            Argument: --delay-days
-Environment Variable: WATCHTOWER_DELAY_DAYS
+            Argument: --defer-days
+Environment Variable: WATCHTOWER_DEFER_DAYS
                 Type: Integer
              Default: false
 ```

internal/actions/mocks/container.go

@@ -79,6 +79,13 @@ func CreateMockContainerWithDigest(id string, name string, image string, created
 	return c
 }
 
+// CreateMockContainerWithDigest should only be used for testing
+func CreateMockContainerWithImageCreatedTime(id string, name string, image string, created time.Time, imageCreated time.Time) wt.Container {
+	c := CreateMockContainer(id, name, image, created)
+	c.ImageInfo().Created = imageCreated.UTC().Format(time.RFC3339Nano)
+	return c
+}
+
 // CreateMockContainerWithConfig creates a container substitute valid for testing
 func CreateMockContainerWithConfig(id string, name string, image string, running bool, restarting bool, created time.Time, config *dockerContainer.Config) wt.Container {
 	content := types.ContainerJSON{

internal/actions/update.go

@@ -2,7 +2,6 @@ package actions
 
 import (
 	"errors"
-	"strings"
 	"time"
 
 	"github.com/containrrr/watchtower/internal/util"
@@ -38,16 +37,17 @@ func Update(client container.Client, params types.UpdateParams) (types.Report, e
 		// stale will be true if there is a more recent image than the current container is using
 		stale, newestImage, err := client.IsContainerStale(targetContainer, params)
 		shouldUpdate := stale && !params.NoRestart && !targetContainer.IsMonitorOnly(params)
-		imageUpdateDelayResolved := true
+		imageUpdateDeferred := false
 		imageAgeDays := 0
 		if err == nil && shouldUpdate {
 			// Check to make sure we have all the necessary information for recreating the container, including ImageInfo
 			err = targetContainer.VerifyConfiguration()
 			if err == nil {
-				if params.DelayDays > 0 {
-					imageAgeDays, err := getImageAgeDays(targetContainer.ImageInfo().Created)
+				if params.DeferDays > 0 {
+					imageAgeDays, imageErr := getImageAgeDays(targetContainer.ImageInfo().Created)
+					err = imageErr
 					if err == nil {
-						imageUpdateDelayResolved = imageAgeDays >= params.DelayDays
+						imageUpdateDeferred = imageAgeDays < params.DeferDays
 					}
 				}
 			} else if log.IsLevelEnabled(log.TraceLevel) {
@@ -66,11 +66,12 @@ func Update(client container.Client, params types.UpdateParams) (types.Report, e
 			stale = false
 			staleCheckFailed++
 			progress.AddSkipped(targetContainer, err)
-		} else if !imageUpdateDelayResolved {
-			log.Infof("New image found for %s that was created %d day(s) ago but update delayed until %d day(s) after creation", targetContainer.Name(), imageAgeDays, params.DelayDays)
+		} else if imageUpdateDeferred {
+			log.Infof("New image found for %s that was created %d day(s) ago but update deferred until %d day(s) after creation", targetContainer.Name(), imageAgeDays, params.DeferDays)
 			// technically the container is stale but we set it to false here because it is this stale flag that tells downstream methods whether to perform the update
 			stale = false
 			progress.AddScanned(targetContainer, newestImage)
+			progress.MarkDeferred(targetContainer.ID())
 		} else {
 			progress.AddScanned(targetContainer, newestImage)
 		}
@@ -90,9 +91,11 @@ func Update(client container.Client, params types.UpdateParams) (types.Report, e
 
 	// containersToUpdate will contain all containers, not just those that need to be updated. The "stale" flag is checked via container.ToRestart()
 	// within stopContainersInReversedOrder and restartContainersInSortedOrder to skip containers with stale set to false (unless LinkedToRestarting set)
+	// NOTE: This logic is changing with latest PR on main repo
 	var containersToUpdate []types.Container
 	for _, c := range containers {
-		if !c.IsMonitorOnly(params) {
+		// pulling this change in from PR 1895 for now to avoid updating status incorrectly
+		if c.ToRestart() && !c.IsMonitorOnly(params) {
 			containersToUpdate = append(containersToUpdate, c)
 			progress.MarkForUpdate(c.ID())
 		}
@@ -286,19 +289,10 @@ func linkedContainerMarkedForRestart(links []string, containers []types.Containe
 }
 
 // Finds the difference between now and a given date, in full days. Input date is expected to originate
-// from an image's Created attribute in ISO 8601, but since these do not always contain the same number of
-// digits for milliseconds, the function also accounts for variations.
+// from an image's Created attribute which will follow ISO 3339/8601 format.
+// Reference: https://docs.docker.com/engine/api/v1.43/#tag/Image/operation/ImageInspect
 func getImageAgeDays(imageCreatedDateTime string) (int, error) {
-
-	// Date strings sometimes vary in how many digits after the decimal point are present. If present, drop millisecond portion to standardize.
-	dotIndex := strings.Index(imageCreatedDateTime, ".")
-	if dotIndex != -1 {
-		imageCreatedDateTime = imageCreatedDateTime[:dotIndex] + "Z"
-	}
-
-	// Define the layout string for the date format without milliseconds
-	layout := "2006-01-02T15:04:05Z"
-	imageCreatedDate, error := time.Parse(layout, imageCreatedDateTime)
+	imageCreatedDate, error := time.Parse(time.RFC3339Nano, imageCreatedDateTime)
 
 	if error != nil {
 		log.Errorf("Error parsing imageCreatedDateTime date (%s). Error: %s", imageCreatedDateTime, error)

internal/actions/update_test.go

@@ -65,6 +65,35 @@ func getLinkedTestData(withImageInfo bool) *TestData {
 	}
 }
 
+func getMixedAgeTestData(keepContainer string) *TestData {
+	return &TestData{
+		NameOfContainerToKeep: keepContainer,
+		Containers: []types.Container{
+			// new container with 5 day old image
+			CreateMockContainerWithImageCreatedTime(
+				"test-container-01",
+				"test-container-01",
+				"fake-image-01:latest",
+				time.Now(),
+				time.Now().AddDate(0, 0, -5)),
+			// new container with 1 day old image
+			CreateMockContainerWithImageCreatedTime(
+				"test-container-02",
+				"test-container-02",
+				"fake-image-02:latest",
+				time.Now(),
+				time.Now().AddDate(0, 0, -1)),
+			// new container with 1 hour old image
+			CreateMockContainerWithImageCreatedTime(
+				"test-container-03",
+				"test-container-03",
+				"fake-image-03:latest",
+				time.Now(),
+				time.Now().Add(-1*time.Hour)),
+		},
+	}
+}
+
 var _ = Describe("the update action", func() {
 	When("watchtower has been instructed to clean up", func() {
 		When("there are multiple containers using the same image", func() {
@@ -258,6 +287,37 @@ var _ = Describe("the update action", func() {
 		})
 	})
 
+	When("watchtower has been instructed to defer updates by some number of days", func() {
+		It("should only update the 1 container with image at least 2 days old when DeferDays is 2", func() {
+			client := CreateMockClient(getMixedAgeTestData(""), false, false)
+			report, err := actions.Update(client, types.UpdateParams{DeferDays: 2})
+			Expect(err).NotTo(HaveOccurred())
+			Expect(report.Updated()).To(HaveLen(1))
+			Expect(report.Deferred()).To(HaveLen(2))
+		})
+		It("should only update the 2 containers with image at least 1 day old when DeferDays is 1", func() {
+			client := CreateMockClient(getMixedAgeTestData(""), false, false)
+			report, err := actions.Update(client, types.UpdateParams{DeferDays: 1})
+			Expect(err).NotTo(HaveOccurred())
+			Expect(report.Updated()).To(HaveLen(2))
+			Expect(report.Deferred()).To(HaveLen(1))
+		})
+		It("should update all containers when DeferDays is 0", func() {
+			client := CreateMockClient(getMixedAgeTestData(""), false, false)
+			report, err := actions.Update(client, types.UpdateParams{DeferDays: 0})
+			Expect(err).NotTo(HaveOccurred())
+			Expect(report.Updated()).To(HaveLen(3))
+			Expect(report.Deferred()).To(HaveLen(0))
+		})
+		It("should update all containers when DeferDays is not specified", func() {
+			client := CreateMockClient(getMixedAgeTestData(""), false, false)
+			report, err := actions.Update(client, types.UpdateParams{})
+			Expect(err).NotTo(HaveOccurred())
+			Expect(report.Updated()).To(HaveLen(3))
+			Expect(report.Deferred()).To(HaveLen(0))
+		})
+	})
+
 	When("watchtower has been instructed to run lifecycle hooks", func() {
 
 		When("pre-update script returns 1", func() {

internal/flags/flags.go

@@ -148,9 +148,9 @@ func RegisterSystemFlags(rootCmd *cobra.Command) {
 		"Enable the execution of commands triggered by pre- and post-update lifecycle hooks")
 
 	flags.IntP(
-		"delay-days",
+		"defer-days",
 		"0",
-		envInt("WATCHTOWER_DELAY_DAYS"),
+		envInt("WATCHTOWER_DEFER_DAYS"),
 		"Number of days to wait for new image version to be in place prior to installing it")
 
 	flags.BoolP(

pkg/metrics/metrics.go

@@ -10,28 +10,31 @@ var metrics *Metrics
 
 // Metric is the data points of a single scan
 type Metric struct {
-	Scanned int
-	Updated int
-	Failed  int
+	Scanned  int
+	Updated  int
+	Deferred int
+	Failed   int
 }
 
 // Metrics is the handler processing all individual scan metrics
 type Metrics struct {
-	channel chan *Metric
-	scanned prometheus.Gauge
-	updated prometheus.Gauge
-	failed  prometheus.Gauge
-	total   prometheus.Counter
-	skipped prometheus.Counter
+	channel  chan *Metric
+	scanned  prometheus.Gauge
+	updated  prometheus.Gauge
+	deferred prometheus.Gauge
+	failed   prometheus.Gauge
+	total    prometheus.Counter
+	skipped  prometheus.Counter
 }
 
 // NewMetric returns a Metric with the counts taken from the appropriate types.Report fields
 func NewMetric(report types.Report) *Metric {
 	return &Metric{
 		Scanned: len(report.Scanned()),
 		// Note: This is for backwards compatibility. ideally, stale containers should be counted separately
-		Updated: len(report.Updated()) + len(report.Stale()),
-		Failed:  len(report.Failed()),
+		Updated:  len(report.Updated()) + len(report.Stale()),
+		Deferred: len(report.Deferred()),
+		Failed:   len(report.Failed()),
 	}
 }
 
@@ -60,6 +63,10 @@ func Default() *Metrics {
 			Name: "watchtower_containers_updated",
 			Help: "Number of containers updated by watchtower during the last scan",
 		}),
+		deferred: promauto.NewGauge(prometheus.GaugeOpts{
+			Name: "watchtower_containers_deferred",
+			Help: "Number of containers deferred by watchtower during the last scan",
+		}),
 		failed: promauto.NewGauge(prometheus.GaugeOpts{
 			Name: "watchtower_containers_failed",
 			Help: "Number of containers where update failed during the last scan",
@@ -95,13 +102,15 @@ func (metrics *Metrics) HandleUpdate(channel <-chan *Metric) {
 			metrics.skipped.Inc()
 			metrics.scanned.Set(0)
 			metrics.updated.Set(0)
+			metrics.deferred.Set(0)
 			metrics.failed.Set(0)
 			continue
 		}
 		// Update metrics with the new values
 		metrics.total.Inc()
 		metrics.scanned.Set(float64(change.Scanned))
 		metrics.updated.Set(float64(change.Updated))
+		metrics.deferred.Set(float64(change.Deferred))
 		metrics.failed.Set(float64(change.Failed))
 	}
 }

pkg/notifications/json.go

@@ -23,12 +23,13 @@ func (d Data) MarshalJSON() ([]byte, error) {
 	var report jsonMap
 	if d.Report != nil {
 		report = jsonMap{
-			`scanned`: marshalReports(d.Report.Scanned()),
-			`updated`: marshalReports(d.Report.Updated()),
-			`failed`:  marshalReports(d.Report.Failed()),
-			`skipped`: marshalReports(d.Report.Skipped()),
-			`stale`:   marshalReports(d.Report.Stale()),
-			`fresh`:   marshalReports(d.Report.Fresh()),
+			`scanned`:  marshalReports(d.Report.Scanned()),
+			`updated`:  marshalReports(d.Report.Updated()),
+			`deferred`: marshalReports(d.Report.Deferred()),
+			`failed`:   marshalReports(d.Report.Failed()),
+			`skipped`:  marshalReports(d.Report.Skipped()),
+			`stale`:    marshalReports(d.Report.Stale()),
+			`fresh`:    marshalReports(d.Report.Fresh()),
 		}
 	}
 

pkg/notifications/json_test.go

@@ -21,6 +21,7 @@ var _ = Describe("JSON template", func() {
 		],
 		"host": "Mock",
 		"report": {
+		"deferred": [],
 		"failed": [
 			{
 				"currentImageId": "01d210000000",
@@ -110,7 +111,7 @@ var _ = Describe("JSON template", func() {
 		},
 		"title": "Watchtower updates on Mock"
 }`
-				data := mockDataFromStates(s.UpdatedState, s.FreshState, s.FailedState, s.SkippedState, s.UpdatedState)
+				data := mockDataFromStates(s.UpdatedState, s.DeferredState, s.FreshState, s.FailedState, s.SkippedState, s.UpdatedState)
 				Expect(getTemplatedResult(`json.v1`, false, data)).To(MatchJSON(expected))
 			})
 		})

pkg/notifications/preview/data/data.go

@@ -72,6 +72,8 @@ func (pb *previewData) addContainer(c containerStatus) {
 		pb.report.scanned = append(pb.report.scanned, &c)
 	case UpdatedState:
 		pb.report.updated = append(pb.report.updated, &c)
+	case DeferredState:
+		pb.report.deferred = append(pb.report.deferred, &c)
 	case FailedState:
 		pb.report.failed = append(pb.report.failed, &c)
 	case SkippedState: