Delete corrupt and exist lower layer when pull image for overlay2 driver

If lower layer corrupt with disk full, host or daemon crash or other reasons,
may let the layer link and lower file empty in the disk, which can lead new
image pull also get an invalid lower file, typically image lower file's
content ends with an colon. Despit the image pull success, but can not inspect
with error message "Error response from daemon: readlink /var/lib/docker/overlay2/l: invalid argument".
So, when pull image check exist layer, can add layer valid logic. If
cause a corrupt layer, delete it and pull it again.

Signed-off-by: Jeff Zvier <zvier20@gmail.com>

daemon/graphdriver/overlay2/overlay.go

@@ -663,10 +663,40 @@ func (d *Driver) Put(id string) error {
 	return nil
 }
 
-// Exists checks to see if the id is already mounted.
+// Exists checks to see if the id is already mounted and valid.
 func (d *Driver) Exists(id string) bool {
-	_, err := os.Stat(d.dir(id))
-	return err == nil
+	dir := d.dir(id)
+	_, err := os.Stat(dir)
+	if err != nil {
+		return false
+	}
+
+	// Check link file, link file must exist and has content.
+	link, err := os.ReadFile(path.Join(dir, "link"))
+	if err != nil {
+		return false
+	}
+	if len(link) == 0 {
+		return false
+	}
+
+	// Check lower file, lower file must has content if exist.
+	lowerFilePath := path.Join(dir, lowerFile)
+	_, err = os.Stat(lowerFilePath)
+	if os.IsNotExist(err) {
+		return true
+	}
+	if err != nil {
+		return false
+	}
+	lower, err := os.ReadFile(lowerFilePath)
+	if err != nil {
+		return false
+	}
+	if len(lower) == 0 {
+		return false
+	}
+	return true
 }
 
 // isParent determines whether the given parent is the direct parent of the

layer/layer_store.go

@@ -398,10 +398,25 @@ func (ls *layerStore) Get(l ChainID) (Layer, error) {
 	if layer == nil {
 		return nil, ErrLayerDoesNotExist
 	}
+	if ls.driver.String() == "overlay2" && !ls.driver.Exists(layer.cacheID) {
+		logrus.Infof("delete corrupt layer with cached id %s", layer.cacheID)
+		if err := ls.deleteCorruptLayer(layer); err != nil {
+			logrus.Errorf("delete corrupt layer with cached id %s failed %s", layer.cacheID, err.Error())
+			return nil, err
+		}
+		return nil, ErrLayerDoesNotExist
+	}
 
 	return layer.getReference(), nil
 }
 
+func (ls *layerStore) deleteCorruptLayer(l *roLayer) error {
+	delete(ls.layerMap, l.chainID)
+	var metadata Metadata
+	err := ls.deleteLayer(l, &metadata)
+	return err
+}
+
 func (ls *layerStore) Map() map[ChainID]Layer {
 	ls.layerL.Lock()
 	defer ls.layerL.Unlock()