Permalink
Browse files

Vendor containers/storage, update containers/image

Vendor containers/storage, and its dependencies github.com/pborman/uuid
and github.com/mistifyio/go-zfs, which we didn't already use.

Update the build Dockerfile to install their dependencies.

Add scriptlets that try to detect whether or not we need to use the
"libdm_no_deferred_remove" and/or "btrfs_noversion" build tags.

Signed-off-by: Nalin Dahyabhai <nalin@redhat.com>
  • Loading branch information...
1 parent 9ba6dd7 commit 4960f390e2ffc3705df7b638eec37026610edbca @nalind nalind committed Aug 31, 2016
Showing with 28,030 additions and 73 deletions.
  1. +3 −0 Dockerfile
  2. +1 −1 Dockerfile.build
  3. +5 −0 Makefile
  4. +7 −0 hack/btrfs_tag.sh
  5. +14 −0 hack/libdm_tag.sh
  6. +4 −0 hack/vendor.sh
  7. +100 −64 vendor/github.com/containers/image/copy/copy.go
  8. +19 −0 vendor/github.com/containers/image/directory/directory_dest.go
  9. +20 −3 vendor/github.com/containers/image/docker/daemon/daemon_dest.go
  10. +2 −2 vendor/github.com/containers/image/docker/docker_client.go
  11. +34 −1 vendor/github.com/containers/image/docker/docker_image_dest.go
  12. +9 −0 vendor/github.com/containers/image/image/memory.go
  13. +5 −0 vendor/github.com/containers/image/image/sourced.go
  14. +22 −0 vendor/github.com/containers/image/oci/layout/oci_dest.go
  15. +8 −0 vendor/github.com/containers/image/openshift/openshift.go
  16. +0 −2 vendor/github.com/containers/image/signature/policy_config.go
  17. +570 −0 vendor/github.com/containers/image/storage/storage_image.go
  18. +127 −0 vendor/github.com/containers/image/storage/storage_reference.go
  19. +285 −0 vendor/github.com/containers/image/storage/storage_transport.go
  20. +2 −0 vendor/github.com/containers/image/transports/transports.go
  21. +14 −0 vendor/github.com/containers/image/types/types.go
  22. +191 −0 vendor/github.com/containers/storage/LICENSE
  23. +586 −0 vendor/github.com/containers/storage/drivers/aufs/aufs.go
  24. +64 −0 vendor/github.com/containers/storage/drivers/aufs/dirs.go
  25. +21 −0 vendor/github.com/containers/storage/drivers/aufs/mount.go
  26. +7 −0 vendor/github.com/containers/storage/drivers/aufs/mount_linux.go
  27. +12 −0 vendor/github.com/containers/storage/drivers/aufs/mount_unsupported.go
  28. +520 −0 vendor/github.com/containers/storage/drivers/btrfs/btrfs.go
  29. +3 −0 vendor/github.com/containers/storage/drivers/btrfs/dummy_unsupported.go
  30. +26 −0 vendor/github.com/containers/storage/drivers/btrfs/version.go
  31. +14 −0 vendor/github.com/containers/storage/drivers/btrfs/version_none.go
  32. +67 −0 vendor/github.com/containers/storage/drivers/counter.go
  33. +2,661 −0 vendor/github.com/containers/storage/drivers/devmapper/deviceset.go
  34. +106 −0 vendor/github.com/containers/storage/drivers/devmapper/devmapper_doc.go
  35. +226 −0 vendor/github.com/containers/storage/drivers/devmapper/driver.go
  36. +89 −0 vendor/github.com/containers/storage/drivers/devmapper/mount.go
  37. +243 −0 vendor/github.com/containers/storage/drivers/driver.go
  38. +19 −0 vendor/github.com/containers/storage/drivers/driver_freebsd.go
  39. +133 −0 vendor/github.com/containers/storage/drivers/driver_linux.go
  40. +65 −0 vendor/github.com/containers/storage/drivers/driver_solaris.go
  41. +15 −0 vendor/github.com/containers/storage/drivers/driver_unsupported.go
  42. +14 −0 vendor/github.com/containers/storage/drivers/driver_windows.go
  43. +157 −0 vendor/github.com/containers/storage/drivers/fsdiff.go
  44. +169 −0 vendor/github.com/containers/storage/drivers/overlay/copy.go
  45. +450 −0 vendor/github.com/containers/storage/drivers/overlay/overlay.go
  46. +3 −0 vendor/github.com/containers/storage/drivers/overlay/overlay_unsupported.go
  47. +88 −0 vendor/github.com/containers/storage/drivers/overlay2/mount.go
  48. +514 −0 vendor/github.com/containers/storage/drivers/overlay2/overlay.go
  49. +3 −0 vendor/github.com/containers/storage/drivers/overlay2/overlay_unsupported.go
  50. +80 −0 vendor/github.com/containers/storage/drivers/overlay2/randomid.go
  51. +32 −0 vendor/github.com/containers/storage/drivers/plugin.go
  52. +7 −0 vendor/github.com/containers/storage/drivers/plugin_unsupported.go
  53. +225 −0 vendor/github.com/containers/storage/drivers/proxy.go
  54. +8 −0 vendor/github.com/containers/storage/drivers/register/register_aufs.go
  55. +8 −0 vendor/github.com/containers/storage/drivers/register/register_btrfs.go
  56. +8 −0 vendor/github.com/containers/storage/drivers/register/register_devicemapper.go
  57. +9 −0 vendor/github.com/containers/storage/drivers/register/register_overlay.go
  58. +6 −0 vendor/github.com/containers/storage/drivers/register/register_vfs.go
  59. +6 −0 vendor/github.com/containers/storage/drivers/register/register_windows.go
  60. +8 −0 vendor/github.com/containers/storage/drivers/register/register_zfs.go
  61. +145 −0 vendor/github.com/containers/storage/drivers/vfs/driver.go
  62. +2 −0 vendor/github.com/containers/storage/drivers/zfs/MAINTAINERS
  63. +405 −0 vendor/github.com/containers/storage/drivers/zfs/zfs.go
  64. +38 −0 vendor/github.com/containers/storage/drivers/zfs/zfs_freebsd.go
  65. +27 −0 vendor/github.com/containers/storage/drivers/zfs/zfs_linux.go
  66. +59 −0 vendor/github.com/containers/storage/drivers/zfs/zfs_solaris.go
  67. +11 −0 vendor/github.com/containers/storage/drivers/zfs/zfs_unsupported.go
  68. +1,147 −0 vendor/github.com/containers/storage/pkg/archive/archive.go
  69. +91 −0 vendor/github.com/containers/storage/pkg/archive/archive_linux.go
  70. +7 −0 vendor/github.com/containers/storage/pkg/archive/archive_other.go
  71. +112 −0 vendor/github.com/containers/storage/pkg/archive/archive_unix.go
  72. +70 −0 vendor/github.com/containers/storage/pkg/archive/archive_windows.go
  73. +446 −0 vendor/github.com/containers/storage/pkg/archive/changes.go
  74. +312 −0 vendor/github.com/containers/storage/pkg/archive/changes_linux.go
  75. +97 −0 vendor/github.com/containers/storage/pkg/archive/changes_other.go
  76. +36 −0 vendor/github.com/containers/storage/pkg/archive/changes_unix.go
  77. +30 −0 vendor/github.com/containers/storage/pkg/archive/changes_windows.go
  78. +458 −0 vendor/github.com/containers/storage/pkg/archive/copy.go
  79. +11 −0 vendor/github.com/containers/storage/pkg/archive/copy_unix.go
  80. +9 −0 vendor/github.com/containers/storage/pkg/archive/copy_windows.go
  81. +279 −0 vendor/github.com/containers/storage/pkg/archive/diff.go
  82. +97 −0 vendor/github.com/containers/storage/pkg/archive/example_changes.go
  83. +16 −0 vendor/github.com/containers/storage/pkg/archive/time_linux.go
  84. +16 −0 vendor/github.com/containers/storage/pkg/archive/time_unsupported.go
  85. +23 −0 vendor/github.com/containers/storage/pkg/archive/whiteouts.go
  86. +59 −0 vendor/github.com/containers/storage/pkg/archive/wrap.go
  87. +97 −0 vendor/github.com/containers/storage/pkg/chrootarchive/archive.go
  88. +86 −0 vendor/github.com/containers/storage/pkg/chrootarchive/archive_unix.go
  89. +22 −0 vendor/github.com/containers/storage/pkg/chrootarchive/archive_windows.go
  90. +103 −0 vendor/github.com/containers/storage/pkg/chrootarchive/chroot_linux.go
  91. +12 −0 vendor/github.com/containers/storage/pkg/chrootarchive/chroot_unix.go
  92. +19 −0 vendor/github.com/containers/storage/pkg/chrootarchive/diff.go
  93. +120 −0 vendor/github.com/containers/storage/pkg/chrootarchive/diff_unix.go
  94. +44 −0 vendor/github.com/containers/storage/pkg/chrootarchive/diff_windows.go
  95. +28 −0 vendor/github.com/containers/storage/pkg/chrootarchive/init_unix.go
  96. +4 −0 vendor/github.com/containers/storage/pkg/chrootarchive/init_windows.go
  97. +807 −0 vendor/github.com/containers/storage/pkg/devicemapper/devmapper.go
  98. +35 −0 vendor/github.com/containers/storage/pkg/devicemapper/devmapper_log.go
  99. +251 −0 vendor/github.com/containers/storage/pkg/devicemapper/devmapper_wrapper.go
  100. +34 −0 vendor/github.com/containers/storage/pkg/devicemapper/devmapper_wrapper_deferred_remove.go
  101. +15 −0 vendor/github.com/containers/storage/pkg/devicemapper/devmapper_wrapper_no_deferred_remove.go
  102. +27 −0 vendor/github.com/containers/storage/pkg/devicemapper/ioctl.go
  103. +11 −0 vendor/github.com/containers/storage/pkg/devicemapper/log.go
  104. +26 −0 vendor/github.com/containers/storage/pkg/directory/directory.go
  105. +48 −0 vendor/github.com/containers/storage/pkg/directory/directory_unix.go
  106. +37 −0 vendor/github.com/containers/storage/pkg/directory/directory_windows.go
  107. +283 −0 vendor/github.com/containers/storage/pkg/fileutils/fileutils.go
  108. +27 −0 vendor/github.com/containers/storage/pkg/fileutils/fileutils_darwin.go
  109. +7 −0 vendor/github.com/containers/storage/pkg/fileutils/fileutils_solaris.go
  110. +22 −0 vendor/github.com/containers/storage/pkg/fileutils/fileutils_unix.go
  111. +7 −0 vendor/github.com/containers/storage/pkg/fileutils/fileutils_windows.go
  112. +39 −0 vendor/github.com/containers/storage/pkg/homedir/homedir.go
  113. +197 −0 vendor/github.com/containers/storage/pkg/idtools/idtools.go
  114. +60 −0 vendor/github.com/containers/storage/pkg/idtools/idtools_unix.go
  115. +18 −0 vendor/github.com/containers/storage/pkg/idtools/idtools_windows.go
  116. +188 −0 vendor/github.com/containers/storage/pkg/idtools/usergroupadd_linux.go
  117. +12 −0 vendor/github.com/containers/storage/pkg/idtools/usergroupadd_unsupported.go
  118. +51 −0 vendor/github.com/containers/storage/pkg/ioutils/buffer.go
  119. +22 −0 vendor/github.com/containers/storage/pkg/ioutils/fmt.go
  120. +82 −0 vendor/github.com/containers/storage/pkg/ioutils/fswriters.go
  121. +226 −0 vendor/github.com/containers/storage/pkg/ioutils/multireader.go
  122. +154 −0 vendor/github.com/containers/storage/pkg/ioutils/readers.go
  123. +10 −0 vendor/github.com/containers/storage/pkg/ioutils/temp_unix.go
  124. +18 −0 vendor/github.com/containers/storage/pkg/ioutils/temp_windows.go
  125. +92 −0 vendor/github.com/containers/storage/pkg/ioutils/writeflusher.go
  126. +66 −0 vendor/github.com/containers/storage/pkg/ioutils/writers.go
  127. +137 −0 vendor/github.com/containers/storage/pkg/loopback/attach_loopback.go
  128. +53 −0 vendor/github.com/containers/storage/pkg/loopback/ioctl.go
  129. +52 −0 vendor/github.com/containers/storage/pkg/loopback/loop_wrapper.go
  130. +63 −0 vendor/github.com/containers/storage/pkg/loopback/loopback.go
  131. +27 −0 vendor/github.com/containers/storage/pkg/mflag/LICENSE
  132. +149 −0 vendor/github.com/containers/storage/pkg/mount/flags.go
  133. +48 −0 vendor/github.com/containers/storage/pkg/mount/flags_freebsd.go
  134. +85 −0 vendor/github.com/containers/storage/pkg/mount/flags_linux.go
  135. +30 −0 vendor/github.com/containers/storage/pkg/mount/flags_unsupported.go
  136. +74 −0 vendor/github.com/containers/storage/pkg/mount/mount.go
  137. +59 −0 vendor/github.com/containers/storage/pkg/mount/mounter_freebsd.go
  138. +21 −0 vendor/github.com/containers/storage/pkg/mount/mounter_linux.go
  139. +33 −0 vendor/github.com/containers/storage/pkg/mount/mounter_solaris.go
  140. +11 −0 vendor/github.com/containers/storage/pkg/mount/mounter_unsupported.go
  141. +40 −0 vendor/github.com/containers/storage/pkg/mount/mountinfo.go
  142. +41 −0 vendor/github.com/containers/storage/pkg/mount/mountinfo_freebsd.go
  143. +95 −0 vendor/github.com/containers/storage/pkg/mount/mountinfo_linux.go
  144. +37 −0 vendor/github.com/containers/storage/pkg/mount/mountinfo_solaris.go
  145. +12 −0 vendor/github.com/containers/storage/pkg/mount/mountinfo_unsupported.go
  146. +6 −0 vendor/github.com/containers/storage/pkg/mount/mountinfo_windows.go
  147. +69 −0 vendor/github.com/containers/storage/pkg/mount/sharedsubtree_linux.go
  148. +74 −0 vendor/github.com/containers/storage/pkg/parsers/kernel/kernel.go
  149. +56 −0 vendor/github.com/containers/storage/pkg/parsers/kernel/kernel_darwin.go
  150. +30 −0 vendor/github.com/containers/storage/pkg/parsers/kernel/kernel_unix.go
  151. +69 −0 vendor/github.com/containers/storage/pkg/parsers/kernel/kernel_windows.go
  152. +19 −0 vendor/github.com/containers/storage/pkg/parsers/kernel/uname_linux.go
  153. +14 −0 vendor/github.com/containers/storage/pkg/parsers/kernel/uname_solaris.go
  154. +18 −0 vendor/github.com/containers/storage/pkg/parsers/kernel/uname_unsupported.go
  155. +69 −0 vendor/github.com/containers/storage/pkg/parsers/parsers.go
  156. +119 −0 vendor/github.com/containers/storage/pkg/pools/pools.go
  157. +11 −0 vendor/github.com/containers/storage/pkg/promise/promise.go
  158. +71 −0 vendor/github.com/containers/storage/pkg/random/random.go
  159. +28 −0 vendor/github.com/containers/storage/pkg/reexec/command_linux.go
  160. +23 −0 vendor/github.com/containers/storage/pkg/reexec/command_unix.go
  161. +12 −0 vendor/github.com/containers/storage/pkg/reexec/command_unsupported.go
  162. +23 −0 vendor/github.com/containers/storage/pkg/reexec/command_windows.go
  163. +47 −0 vendor/github.com/containers/storage/pkg/reexec/reexec.go
  164. +71 −0 vendor/github.com/containers/storage/pkg/stringid/stringid.go
  165. +52 −0 vendor/github.com/containers/storage/pkg/system/chtimes.go
  166. +14 −0 vendor/github.com/containers/storage/pkg/system/chtimes_unix.go
  167. +27 −0 vendor/github.com/containers/storage/pkg/system/chtimes_windows.go
  168. +10 −0 vendor/github.com/containers/storage/pkg/system/errors.go
  169. +83 −0 vendor/github.com/containers/storage/pkg/system/events_windows.go
  170. +19 −0 vendor/github.com/containers/storage/pkg/system/filesys.go
  171. +82 −0 vendor/github.com/containers/storage/pkg/system/filesys_windows.go
  172. +19 −0 vendor/github.com/containers/storage/pkg/system/lstat.go
  173. +25 −0 vendor/github.com/containers/storage/pkg/system/lstat_windows.go
  174. +17 −0 vendor/github.com/containers/storage/pkg/system/meminfo.go
  175. +65 −0 vendor/github.com/containers/storage/pkg/system/meminfo_linux.go
  176. +128 −0 vendor/github.com/containers/storage/pkg/system/meminfo_solaris.go
  177. +8 −0 vendor/github.com/containers/storage/pkg/system/meminfo_unsupported.go
  178. +44 −0 vendor/github.com/containers/storage/pkg/system/meminfo_windows.go
  179. +22 −0 vendor/github.com/containers/storage/pkg/system/mknod.go
  180. +13 −0 vendor/github.com/containers/storage/pkg/system/mknod_windows.go
  181. +14 −0 vendor/github.com/containers/storage/pkg/system/path_unix.go
  182. +37 −0 vendor/github.com/containers/storage/pkg/system/path_windows.go
  183. +53 −0 vendor/github.com/containers/storage/pkg/system/stat.go
  184. +27 −0 vendor/github.com/containers/storage/pkg/system/stat_freebsd.go
  185. +33 −0 vendor/github.com/containers/storage/pkg/system/stat_linux.go
  186. +15 −0 vendor/github.com/containers/storage/pkg/system/stat_openbsd.go
  187. +34 −0 vendor/github.com/containers/storage/pkg/system/stat_solaris.go
  188. +17 −0 vendor/github.com/containers/storage/pkg/system/stat_unsupported.go
  189. +43 −0 vendor/github.com/containers/storage/pkg/system/stat_windows.go
  190. +17 −0 vendor/github.com/containers/storage/pkg/system/syscall_unix.go
  191. +103 −0 vendor/github.com/containers/storage/pkg/system/syscall_windows.go
  192. +13 −0 vendor/github.com/containers/storage/pkg/system/umask.go
  193. +9 −0 vendor/github.com/containers/storage/pkg/system/umask_windows.go
  194. +8 −0 vendor/github.com/containers/storage/pkg/system/utimes_darwin.go
  195. +22 −0 vendor/github.com/containers/storage/pkg/system/utimes_freebsd.go
  196. +26 −0 vendor/github.com/containers/storage/pkg/system/utimes_linux.go
  197. +10 −0 vendor/github.com/containers/storage/pkg/system/utimes_unsupported.go
  198. +63 −0 vendor/github.com/containers/storage/pkg/system/xattrs_linux.go
  199. +13 −0 vendor/github.com/containers/storage/pkg/system/xattrs_unsupported.go
  200. +478 −0 vendor/github.com/containers/storage/storage/containers.go
  201. +448 −0 vendor/github.com/containers/storage/storage/images.go
  202. +881 −0 vendor/github.com/containers/storage/storage/layers.go
  203. +138 −0 vendor/github.com/containers/storage/storage/lockfile.go
  204. +2,171 −0 vendor/github.com/containers/storage/storage/store.go
  205. +13 −0 vendor/github.com/containers/storage/storageversion/version_lib.go
  206. +3 −0 vendor/github.com/go-check/check/.travis.yml
  207. +1 −0 vendor/github.com/mistifyio/go-zfs/.gitignore
  208. +41 −0 vendor/github.com/mistifyio/go-zfs/.travis.yml
  209. +60 −0 vendor/github.com/mistifyio/go-zfs/CONTRIBUTING.md
  210. +201 −0 vendor/github.com/mistifyio/go-zfs/LICENSE
  211. +54 −0 vendor/github.com/mistifyio/go-zfs/README.md
  212. +34 −0 vendor/github.com/mistifyio/go-zfs/Vagrantfile
  213. +18 −0 vendor/github.com/mistifyio/go-zfs/error.go
  214. +352 −0 vendor/github.com/mistifyio/go-zfs/utils.go
  215. +17 −0 vendor/github.com/mistifyio/go-zfs/utils_notsolaris.go
  216. +17 −0 vendor/github.com/mistifyio/go-zfs/utils_solaris.go
  217. +451 −0 vendor/github.com/mistifyio/go-zfs/zfs.go
  218. +112 −0 vendor/github.com/mistifyio/go-zfs/zpool.go
  219. +84 −0 vendor/github.com/opencontainers/runc/libcontainer/label/label.go
  220. +208 −0 vendor/github.com/opencontainers/runc/libcontainer/label/label_selinux.go
  221. +143 −0 vendor/github.com/opencontainers/runc/libcontainer/system/linux.go
  222. +43 −0 vendor/github.com/opencontainers/runc/libcontainer/system/proc.go
  223. +40 −0 vendor/github.com/opencontainers/runc/libcontainer/system/setns_linux.go
  224. +25 −0 vendor/github.com/opencontainers/runc/libcontainer/system/syscall_linux_386.go
  225. +25 −0 vendor/github.com/opencontainers/runc/libcontainer/system/syscall_linux_64.go
  226. +25 −0 vendor/github.com/opencontainers/runc/libcontainer/system/syscall_linux_arm.go
  227. +12 −0 vendor/github.com/opencontainers/runc/libcontainer/system/sysconfig.go
  228. +15 −0 vendor/github.com/opencontainers/runc/libcontainer/system/sysconfig_notcgo.go
  229. +9 −0 vendor/github.com/opencontainers/runc/libcontainer/system/unsupported.go
  230. +99 −0 vendor/github.com/opencontainers/runc/libcontainer/system/xattrs_linux.go
  231. +10 −0 vendor/github.com/pborman/uuid/.travis.yml
  232. +1 −0 vendor/github.com/pborman/uuid/CONTRIBUTORS
  233. +27 −0 vendor/github.com/pborman/uuid/LICENSE
  234. +13 −0 vendor/github.com/pborman/uuid/README.md
  235. +84 −0 vendor/github.com/pborman/uuid/dce.go
  236. +8 −0 vendor/github.com/pborman/uuid/doc.go
  237. +53 −0 vendor/github.com/pborman/uuid/hash.go
  238. +34 −0 vendor/github.com/pborman/uuid/json.go
  239. +117 −0 vendor/github.com/pborman/uuid/node.go
  240. +58 −0 vendor/github.com/pborman/uuid/sql.go
  241. +132 −0 vendor/github.com/pborman/uuid/time.go
  242. +43 −0 vendor/github.com/pborman/uuid/util.go
  243. +176 −0 vendor/github.com/pborman/uuid/uuid.go
  244. +41 −0 vendor/github.com/pborman/uuid/version1.go
  245. +25 −0 vendor/github.com/pborman/uuid/version4.go
  246. +329 −0 vendor/github.com/vbatts/tar-split/archive/tar/common.go
  247. +943 −0 vendor/github.com/vbatts/tar-split/archive/tar/reader.go
  248. +20 −0 vendor/github.com/vbatts/tar-split/archive/tar/stat_atim.go
  249. +20 −0 vendor/github.com/vbatts/tar-split/archive/tar/stat_atimespec.go
  250. +32 −0 vendor/github.com/vbatts/tar-split/archive/tar/stat_unix.go
  251. +396 −0 vendor/github.com/vbatts/tar-split/archive/tar/writer.go
  252. +44 −0 vendor/github.com/vbatts/tar-split/tar/asm/README.md
  253. +130 −0 vendor/github.com/vbatts/tar-split/tar/asm/assemble.go
  254. +141 −0 vendor/github.com/vbatts/tar-split/tar/asm/disassemble.go
  255. +9 −0 vendor/github.com/vbatts/tar-split/tar/asm/doc.go
  256. +12 −0 vendor/github.com/vbatts/tar-split/tar/storage/doc.go
  257. +78 −0 vendor/github.com/vbatts/tar-split/tar/storage/entry.go
  258. +104 −0 vendor/github.com/vbatts/tar-split/tar/storage/getter.go
  259. +127 −0 vendor/github.com/vbatts/tar-split/tar/storage/packer.go
View
@@ -1,6 +1,9 @@
FROM fedora
RUN dnf -y update && dnf install -y make git golang golang-github-cpuguy83-go-md2man \
+ # storage deps
+ btrfs-progs-devel \
+ device-mapper-devel \
# gpgme bindings deps
libassuan-devel gpgme-devel \
gnupg \
View
@@ -1,5 +1,5 @@
FROM ubuntu:16.04
RUN apt-get update
-RUN apt-get install -y golang git-core libgpgme11-dev go-md2man
+RUN apt-get install -y golang btrfs-tools git-core libdevmapper-dev libgpgme11-dev go-md2man
ENV GOPATH=/
WORKDIR /src/github.com/projectatomic/skopeo
View
@@ -32,6 +32,11 @@ GIT_COMMIT := $(shell git rev-parse HEAD 2> /dev/null || true)
MANPAGES_MD = $(wildcard docs/*.md)
+BTRFS_BUILD_TAG = $(shell hack/btrfs_tag.sh)
+LIBDM_BUILD_TAG = $(shell hack/libdm_tag.sh)
+LOCAL_BUILD_TAGS = $(BTRFS_BUILD_TAG) $(LIBDM_BUILD_TAG)
+BUILDTAGS += $(LOCAL_BUILD_TAGS)
+
# make all DEBUG=1
# Note: Uses the -N -l go compiler options to disable compiler optimizations
# and inlining. Using these build options allows you to subsequently
View
@@ -0,0 +1,7 @@
+#!/bin/bash
+cc -E - > /dev/null 2> /dev/null << EOF
+#include <btrfs/version.h>
+EOF
+if test $? -ne 0 ; then
+ echo btrfs_noversion
+fi
View
@@ -0,0 +1,14 @@
+#!/bin/bash
+tmpdir="$PWD/tmp.$RANDOM"
+mkdir -p "$tmpdir"
+trap 'rm -fr "$tmpdir"' EXIT
+cc -c -o "$tmpdir"/libdm_tag.o -x c - > /dev/null 2> /dev/null << EOF
+#include <libdevmapper.h>
+int main() {
+ struct dm_task *task;
+ return 0;
+}
+EOF
+if test $? -ne 0 ; then
+ echo libdm_no_deferred_remove
+fi
View
@@ -8,6 +8,7 @@ source 'hack/.vendor-helpers.sh'
clone git github.com/urfave/cli v1.17.0
clone git github.com/containers/image master
clone git gopkg.in/cheggaaa/pb.v1 ad4efe000aa550bb54918c06ebbadc0ff17687b9 https://github.com/cheggaaa/pb
+clone git github.com/containers/storage master
clone git github.com/Sirupsen/logrus v0.10.0
clone git github.com/go-check/check v1
clone git github.com/stretchr/testify v1.1.3
@@ -34,6 +35,9 @@ clone git k8s.io/kubernetes 4a3f9c5b19c7ff804cbc1bf37a15c044ca5d2353 https://git
clone git github.com/ghodss/yaml 73d445a93680fa1a78ae23a5839bad48f32ba1ee
clone git gopkg.in/yaml.v2 d466437aa4adc35830964cffc5b5f262c63ddcb4
clone git github.com/imdario/mergo 6633656539c1639d9d78127b7d47c622b5d7b6dc
+# containers/storage's dependencies that aren't already being pulled in
+clone git github.com/mistifyio/go-zfs 22c9b32c84eb0d0c6f4043b6e90fc94073de92fa
+clone git github.com/pborman/uuid v1.0
clean
@@ -32,6 +32,21 @@ type digestingReader struct {
validationFailed bool
}
+// imageCopier allows us to keep track of diffID values for blobs, and other
+// data, that we're copying between images, and cache other information that
+// might allow us to take some shortcuts
+type imageCopier struct {
+ copiedBlobs map[digest.Digest]digest.Digest
+ cachedDiffIDs map[digest.Digest]digest.Digest
+ manifestUpdates *types.ManifestUpdateOptions
+ dest types.ImageDestination
+ src types.Image
+ rawSource types.ImageSource
+ diffIDsAreNeeded bool
+ canModifyManifest bool
+ reportWriter io.Writer
+}
+
// newDigestingReader returns an io.Reader implementation with contents of source, which will eventually return a non-EOF error
// and set validationFailed to true if the source stream does not match expectedDigest.
func newDigestingReader(source io.Reader, expectedDigest digest.Digest) (*digestingReader, error) {
@@ -147,7 +162,20 @@ func Image(policyContext *signature.PolicyContext, destRef, srcRef types.ImageRe
return err
}
- if err := copyLayers(&manifestUpdates, dest, src, rawSource, canModifyManifest, reportWriter); err != nil {
+ // If src.UpdatedImageNeedsLayerDiffIDs(manifestUpdates) will be true, it needs to be true by the time we get here.
+ ic := imageCopier{
+ copiedBlobs: make(map[digest.Digest]digest.Digest),
+ cachedDiffIDs: make(map[digest.Digest]digest.Digest),
+ manifestUpdates: &manifestUpdates,
+ dest: dest,
+ src: src,
+ rawSource: rawSource,
+ diffIDsAreNeeded: src.UpdatedImageNeedsLayerDiffIDs(manifestUpdates),
+ canModifyManifest: canModifyManifest,
+ reportWriter: reportWriter,
+ }
+
+ if err := ic.copyLayers(); err != nil {
return err
}
@@ -167,7 +195,7 @@ func Image(policyContext *signature.PolicyContext, destRef, srcRef types.ImageRe
return fmt.Errorf("Error reading manifest: %v", err)
}
- if err := copyConfig(dest, pendingImage, reportWriter); err != nil {
+ if err := ic.copyConfig(pendingImage); err != nil {
return err
}
@@ -206,57 +234,41 @@ func Image(policyContext *signature.PolicyContext, destRef, srcRef types.ImageRe
return nil
}
-// copyLayers copies layers from src/rawSource to dest, using and updating manifestUpdates if necessary and canModifyManifest.
-// If src.UpdatedImageNeedsLayerDiffIDs(manifestUpdates) will be true, it needs to be true by the time this function is called.
-func copyLayers(manifestUpdates *types.ManifestUpdateOptions, dest types.ImageDestination, src types.Image, rawSource types.ImageSource,
- canModifyManifest bool, reportWriter io.Writer) error {
- type copiedLayer struct {
- blobInfo types.BlobInfo
- diffID digest.Digest
- }
-
- diffIDsAreNeeded := src.UpdatedImageNeedsLayerDiffIDs(*manifestUpdates)
-
- srcInfos := src.LayerInfos()
+// copyLayers copies layers from src/rawSource to dest, using and updating ic.manifestUpdates if necessary and ic.canModifyManifest.
+func (ic *imageCopier) copyLayers() error {
+ srcInfos := ic.src.LayerInfos()
destInfos := []types.BlobInfo{}
diffIDs := []digest.Digest{}
- copiedLayers := map[digest.Digest]copiedLayer{}
for _, srcLayer := range srcInfos {
- cl, ok := copiedLayers[srcLayer.Digest]
- if !ok {
- var (
- destInfo types.BlobInfo
- diffID digest.Digest
- err error
- )
- if dest.AcceptsForeignLayerURLs() && len(srcLayer.URLs) != 0 {
- // DiffIDs are, currently, needed only when converting from schema1.
- // In which case src.LayerInfos will not have URLs because schema1
- // does not support them.
- if diffIDsAreNeeded {
- return errors.New("getting DiffID for foreign layers is unimplemented")
- }
- destInfo = srcLayer
- fmt.Fprintf(reportWriter, "Skipping foreign layer %q copy to %s\n", destInfo.Digest, dest.Reference().Transport().Name())
- } else {
- fmt.Fprintf(reportWriter, "Copying blob %s\n", srcLayer.Digest)
- destInfo, diffID, err = copyLayer(dest, rawSource, srcLayer, diffIDsAreNeeded, canModifyManifest, reportWriter)
- if err != nil {
- return err
- }
+ var (
+ destInfo types.BlobInfo
+ diffID digest.Digest
+ err error
+ )
+ if ic.dest.AcceptsForeignLayerURLs() && len(srcLayer.URLs) != 0 {
+ // DiffIDs are, currently, needed only when converting from schema1.
+ // In which case src.LayerInfos will not have URLs because schema1
+ // does not support them.
+ if ic.diffIDsAreNeeded {
+ return errors.New("getting DiffID for foreign layers is unimplemented")
+ }
+ destInfo = srcLayer
+ fmt.Fprintf(ic.reportWriter, "Skipping foreign layer %q copy to %s\n", destInfo.Digest, ic.dest.Reference().Transport().Name())
+ } else {
+ destInfo, diffID, err = ic.copyLayer(srcLayer)
+ if err != nil {
+ return err
}
- cl = copiedLayer{blobInfo: destInfo, diffID: diffID}
- copiedLayers[srcLayer.Digest] = cl
}
- destInfos = append(destInfos, cl.blobInfo)
- diffIDs = append(diffIDs, cl.diffID)
+ destInfos = append(destInfos, destInfo)
+ diffIDs = append(diffIDs, diffID)
}
- manifestUpdates.InformationOnly.LayerInfos = destInfos
- if diffIDsAreNeeded {
- manifestUpdates.InformationOnly.LayerDiffIDs = diffIDs
+ ic.manifestUpdates.InformationOnly.LayerInfos = destInfos
+ if ic.diffIDsAreNeeded {
+ ic.manifestUpdates.InformationOnly.LayerDiffIDs = diffIDs
}
if layerDigestsDiffer(srcInfos, destInfos) {
- manifestUpdates.LayerInfos = destInfos
+ ic.manifestUpdates.LayerInfos = destInfos
}
return nil
}
@@ -275,15 +287,15 @@ func layerDigestsDiffer(a, b []types.BlobInfo) bool {
}
// copyConfig copies config.json, if any, from src to dest.
-func copyConfig(dest types.ImageDestination, src types.Image, reportWriter io.Writer) error {
+func (ic *imageCopier) copyConfig(src types.Image) error {
srcInfo := src.ConfigInfo()
if srcInfo.Digest != "" {
- fmt.Fprintf(reportWriter, "Copying config %s\n", srcInfo.Digest)
+ fmt.Fprintf(ic.reportWriter, "Copying config %s\n", srcInfo.Digest)
configBlob, err := src.ConfigBlob()
if err != nil {
return fmt.Errorf("Error reading config blob %s: %v", srcInfo.Digest, err)
}
- destInfo, err := copyBlobFromStream(dest, bytes.NewReader(configBlob), srcInfo, nil, false, reportWriter)
+ destInfo, err := ic.copyBlobFromStream(bytes.NewReader(configBlob), srcInfo, nil, false)
if err != nil {
return err
}
@@ -303,16 +315,40 @@ type diffIDResult struct {
// copyLayer copies a layer with srcInfo (with known Digest and possibly known Size) in src to dest, perhaps compressing it if canCompress,
// and returns a complete blobInfo of the copied layer, and a value for LayerDiffIDs if diffIDIsNeeded
-func copyLayer(dest types.ImageDestination, src types.ImageSource, srcInfo types.BlobInfo,
- diffIDIsNeeded bool, canCompress bool, reportWriter io.Writer) (types.BlobInfo, digest.Digest, error) {
- srcStream, srcBlobSize, err := src.GetBlob(srcInfo)
+func (ic *imageCopier) copyLayer(srcInfo types.BlobInfo) (types.BlobInfo, digest.Digest, error) {
+ // Check if we already have a blob with this digest
+ haveBlob, extantBlobSize, err := ic.dest.HasBlob(srcInfo)
+ if err != nil && err != types.ErrBlobNotFound {
+ return types.BlobInfo{}, "", fmt.Errorf("Error checking for blob %s at destination: %v", srcInfo.Digest, err)
+ }
+ // If we already have a cached diffID for this blob, we don't need to compute it
+ diffIDIsNeeded := ic.diffIDsAreNeeded && (ic.cachedDiffIDs[srcInfo.Digest] == "")
+ // If we already have the blob, and we don't need to recompute the diffID, then we might be able to avoid reading it again
+ if haveBlob && !diffIDIsNeeded {
+ // Check the blob sizes match, if we were given a size this time
+ if srcInfo.Size != -1 && srcInfo.Size != extantBlobSize {
+ return types.BlobInfo{}, "", fmt.Errorf("Error: blob %s is already present, but with size %d instead of %d", srcInfo.Digest, extantBlobSize, srcInfo.Size)
+ }
+ srcInfo.Size = extantBlobSize
+ // Tell the image destination that this blob's delta is being applied again. For some image destinations, this can be faster than using GetBlob/PutBlob
+ blobinfo, err := ic.dest.ReapplyBlob(srcInfo)
+ if err != nil {
+ return types.BlobInfo{}, "", fmt.Errorf("Error reapplying blob %s at destination: %v", srcInfo.Digest, err)
+ }
+ fmt.Fprintf(ic.reportWriter, "Skipping fetch of repeat blob %s\n", srcInfo.Digest)
+ return blobinfo, ic.cachedDiffIDs[srcInfo.Digest], err
+ }
+
+ // Fallback: copy the layer, computing the diffID if we need to do so
+ fmt.Fprintf(ic.reportWriter, "Copying blob %s\n", srcInfo.Digest)
+ srcStream, srcBlobSize, err := ic.rawSource.GetBlob(srcInfo)
if err != nil {
return types.BlobInfo{}, "", fmt.Errorf("Error reading blob %s: %v", srcInfo.Digest, err)
}
defer srcStream.Close()
- blobInfo, diffIDChan, err := copyLayerFromStream(dest, srcStream, types.BlobInfo{Digest: srcInfo.Digest, Size: srcBlobSize},
- diffIDIsNeeded, canCompress, reportWriter)
+ blobInfo, diffIDChan, err := ic.copyLayerFromStream(srcStream, types.BlobInfo{Digest: srcInfo.Digest, Size: srcBlobSize},
+ diffIDIsNeeded)
if err != nil {
return types.BlobInfo{}, "", err
}
@@ -323,6 +359,7 @@ func copyLayer(dest types.ImageDestination, src types.ImageSource, srcInfo types
return types.BlobInfo{}, "", fmt.Errorf("Error computing layer DiffID: %v", diffIDResult.err)
}
logrus.Debugf("Computed DiffID %s for layer %s", diffIDResult.digest, srcInfo.Digest)
+ ic.cachedDiffIDs[srcInfo.Digest] = diffIDResult.digest
}
return blobInfo, diffIDResult.digest, nil
}
@@ -331,8 +368,8 @@ func copyLayer(dest types.ImageDestination, src types.ImageSource, srcInfo types
// it copies a blob with srcInfo (with known Digest and possibly known Size) from srcStream to dest,
// perhaps compressing the stream if canCompress,
// and returns a complete blobInfo of the copied blob and perhaps a <-chan diffIDResult if diffIDIsNeeded, to be read by the caller.
-func copyLayerFromStream(dest types.ImageDestination, srcStream io.Reader, srcInfo types.BlobInfo,
- diffIDIsNeeded bool, canCompress bool, reportWriter io.Writer) (types.BlobInfo, <-chan diffIDResult, error) {
+func (ic *imageCopier) copyLayerFromStream(srcStream io.Reader, srcInfo types.BlobInfo,
+ diffIDIsNeeded bool) (types.BlobInfo, <-chan diffIDResult, error) {
var getDiffIDRecorder func(decompressorFunc) io.Writer // = nil
var diffIDChan chan diffIDResult
@@ -356,8 +393,7 @@ func copyLayerFromStream(dest types.ImageDestination, srcStream io.Reader, srcIn
return pipeWriter
}
}
- blobInfo, err := copyBlobFromStream(dest, srcStream, srcInfo,
- getDiffIDRecorder, canCompress, reportWriter) // Sets err to nil on success
+ blobInfo, err := ic.copyBlobFromStream(srcStream, srcInfo, getDiffIDRecorder, ic.canModifyManifest) // Sets err to nil on success
return blobInfo, diffIDChan, err
// We need the defer … pipeWriter.CloseWithError() to happen HERE so that the caller can block on reading from diffIDChan
}
@@ -391,9 +427,9 @@ func computeDiffID(stream io.Reader, decompressor decompressorFunc) (digest.Dige
// perhaps sending a copy to an io.Writer if getOriginalLayerCopyWriter != nil,
// perhaps compressing it if canCompress,
// and returns a complete blobInfo of the copied blob.
-func copyBlobFromStream(dest types.ImageDestination, srcStream io.Reader, srcInfo types.BlobInfo,
- getOriginalLayerCopyWriter func(decompressor decompressorFunc) io.Writer, canCompress bool,
- reportWriter io.Writer) (types.BlobInfo, error) {
+func (ic *imageCopier) copyBlobFromStream(srcStream io.Reader, srcInfo types.BlobInfo,
+ getOriginalLayerCopyWriter func(decompressor decompressorFunc) io.Writer,
+ canCompress bool) (types.BlobInfo, error) {
// The copying happens through a pipeline of connected io.Readers.
// === Input: srcStream
@@ -419,13 +455,13 @@ func copyBlobFromStream(dest types.ImageDestination, srcStream io.Reader, srcInf
// === Report progress using a pb.Reader.
bar := pb.New(int(srcInfo.Size)).SetUnits(pb.U_BYTES)
- bar.Output = reportWriter
+ bar.Output = ic.reportWriter
bar.SetMaxWidth(80)
bar.ShowTimeLeft = false
bar.ShowPercent = false
bar.Start()
destStream = bar.NewProxyReader(destStream)
- defer fmt.Fprint(reportWriter, "\n")
+ defer fmt.Fprint(ic.reportWriter, "\n")
// === Send a copy of the original, uncompressed, stream, to a separate path if necessary.
var originalLayerReader io.Reader // DO NOT USE this other than to drain the input if no other consumer in the pipeline has done so.
@@ -436,7 +472,7 @@ func copyBlobFromStream(dest types.ImageDestination, srcStream io.Reader, srcInf
// === Compress the layer if it is uncompressed and compression is desired
var inputInfo types.BlobInfo
- if !canCompress || isCompressed || !dest.ShouldCompressLayers() {
+ if !canCompress || isCompressed || !ic.dest.ShouldCompressLayers() {
logrus.Debugf("Using original blob without modification")
inputInfo = srcInfo
} else {
@@ -454,7 +490,7 @@ func copyBlobFromStream(dest types.ImageDestination, srcStream io.Reader, srcInf
}
// === Finally, send the layer stream to dest.
- uploadedInfo, err := dest.PutBlob(destStream, inputInfo)
+ uploadedInfo, err := ic.dest.PutBlob(destStream, inputInfo)
if err != nil {
return types.BlobInfo{}, fmt.Errorf("Error writing blob: %v", err)
}
@@ -94,6 +94,25 @@ func (d *dirImageDestination) PutBlob(stream io.Reader, inputInfo types.BlobInfo
return types.BlobInfo{Digest: computedDigest, Size: size}, nil
}
+func (d *dirImageDestination) HasBlob(info types.BlobInfo) (bool, int64, error) {
+ if info.Digest == "" {
+ return false, -1, fmt.Errorf(`"Can not check for a blob with unknown digest`)
+ }
+ blobPath := d.ref.layerPath(info.Digest)
+ finfo, err := os.Stat(blobPath)
+ if err != nil && os.IsNotExist(err) {
+ return false, -1, types.ErrBlobNotFound
+ }
+ if err != nil {
+ return false, -1, err
+ }
+ return true, finfo.Size(), nil
+}
+
+func (d *dirImageDestination) ReapplyBlob(info types.BlobInfo) (types.BlobInfo, error) {
+ return info, nil
+}
+
func (d *dirImageDestination) PutManifest(manifest []byte) error {
return ioutil.WriteFile(d.ref.manifestPath(), manifest, 0644)
}
Oops, something went wrong.

0 comments on commit 4960f39

Please sign in to comment.