petset and volume debug messages

pkg/volume/gce_pd/attacher.go

@@ -166,6 +166,7 @@ func (attacher *gcePersistentDiskAttacher) MountDevice(spec *volume.Spec, device
 			os.Remove(deviceMountPath)
 			return err
 		}
+		glog.V(4).Infof("formatting spec %v devicePath %v deviceMountPath %v fs %v with options %+v", spec.Name(), devicePath, deviceMountPath, volumeSource.FSType, options)
 	}
 	return nil
 }

pkg/volume/gce_pd/gce_pd.go

@@ -86,11 +86,12 @@ func getVolumeSource(spec *volume.Spec) (*api.GCEPersistentDiskVolumeSource, boo
 	if spec.Volume != nil && spec.Volume.GCEPersistentDisk != nil {
 		volumeSource = spec.Volume.GCEPersistentDisk
 		readOnly = volumeSource.ReadOnly
+		glog.V(4).Infof("volume source %v spec %v, readonly flag retrieved from source: %v", volumeSource.PDName, spec.Name(), readOnly)
 	} else {
 		volumeSource = spec.PersistentVolume.Spec.GCEPersistentDisk
 		readOnly = spec.ReadOnly
+		glog.V(4).Infof("volume source %v spec %v, readonly flag retrieved from spec: %v", volumeSource.PDName, spec.Name(), readOnly)
 	}
-
 	return volumeSource, readOnly
 }
 
@@ -219,7 +220,7 @@ func (b *gcePersistentDiskMounter) SetUp(fsGroup *int64) error {
 func (b *gcePersistentDiskMounter) SetUpAt(dir string, fsGroup *int64) error {
 	// TODO: handle failed mounts here.
 	notMnt, err := b.mounter.IsLikelyNotMountPoint(dir)
-	glog.V(4).Infof("PersistentDisk set up: %s %v %v", dir, !notMnt, err)
+	glog.V(4).Infof("PersistentDisk set up: %s %v %v, pd name %v readOnly %v", dir, !notMnt, err, b.pdName, b.readOnly)
 	if err != nil && !os.IsNotExist(err) {
 		return err
 	}

test/e2e/petset.go

@@ -109,17 +109,23 @@ var _ = framework.KubeDescribe("PetSet [Slow] [Feature:PetSet]", func() {
 			By("Saturating pet set " + ps.Name)
 			pst.saturate(ps)
 
+			By("Verifying petset mounted data directory is usable")
+			ExpectNoError(pst.checkMount(ps, "/data"))
+
 			cmd := "echo $(hostname) > /data/hostname; sync;"
 			By("Running " + cmd + " in all pets")
-			pst.execInPets(ps, cmd)
+			ExpectNoError(pst.execInPets(ps, cmd))
 
 			By("Restarting pet set " + ps.Name)
 			pst.restart(ps)
 			pst.saturate(ps)
 
+			By("Verifying petset mounted data directory is usable")
+			ExpectNoError(pst.checkMount(ps, "/data"))
+
 			cmd = "if [ \"$(cat /data/hostname)\" = \"$(hostname)\" ]; then exit 0; else exit 1; fi"
 			By("Running " + cmd + " in all pets")
-			pst.execInPets(ps, cmd)
+			ExpectNoError(pst.execInPets(ps, cmd))
 		})
 
 		It("should handle healthy pet restarts during scale [Feature:PetSet]", func() {
@@ -414,13 +420,32 @@ func (p *petSetTester) createPetSet(manifestPath, ns string) *apps.PetSet {
 	return ps
 }
 
-func (p *petSetTester) execInPets(ps *apps.PetSet, cmd string) {
+func (p *petSetTester) checkMount(ps *apps.PetSet, mountPath string) error {
+	for _, cmd := range []string{
+		// Print inode, size etc
+		fmt.Sprintf("ls -idlh %v", mountPath),
+		// Print subdirs
+		fmt.Sprintf("find %v", mountPath),
+		// Try writing
+		fmt.Sprintf("touch %v", filepath.Join(mountPath, fmt.Sprintf("%v", time.Now().UnixNano()))),
+	} {
+		if err := p.execInPets(ps, cmd); err != nil {
+			return fmt.Errorf("failed to execute %v, error: %v", cmd, err)
+		}
+	}
+	return nil
+}
+
+func (p *petSetTester) execInPets(ps *apps.PetSet, cmd string) error {
 	podList := p.getPodList(ps)
 	for _, pet := range podList.Items {
 		stdout, err := framework.RunHostCmd(pet.Namespace, pet.Name, cmd)
-		ExpectNoError(err)
 		framework.Logf("stdout of %v on %v: %v", cmd, pet.Name, stdout)
+		if err != nil {
+			return err
+		}
 	}
+	return nil
 }
 
 func (p *petSetTester) saturate(ps *apps.PetSet) {