Skip to content
Permalink
Browse files

blobsfile,...: switched to Zstandard

  • Loading branch information...
tsileo committed Dec 22, 2018
1 parent 765e411 commit 1fa5baf5d4bf449aa74b5ef8da54eb44293967cd
Showing with 20,227 additions and 3,263 deletions.
  1. +28 −10 go.mod
  2. +21 −0 go.sum
  3. +1 −0 integration_tests/blobstore_kvstore.py
  4. +0 −14 pkg/blobstore/api/api.go
  5. +1 −72 pkg/blobstore/blobstore.go
  6. +1 −1 pkg/filetree/filetreeutil/node/node.go
  7. +2 −2 pkg/filetree/writer/dirwriter.go
  8. +1 −1 pkg/gitserver/gitserver.go
  9. +2 −1 pkg/gitserver/lua/lua.go
  10. +0 −8 pkg/stash/stash.go
  11. +0 −13 pkg/stash/store/store.go
  12. +4 −3 vendor/a4.io/blobsfile/README.md
  13. +49 −84 vendor/a4.io/blobsfile/blobsfile.go
  14. +19 −0 vendor/a4.io/blobsfile/go.mod
  15. +141 −0 vendor/a4.io/blobsfile/go.sum
  16. +4 −2 vendor/a4.io/gluapp/.travis.yml
  17. +7 −0 vendor/a4.io/ssse/LICENSE
  18. +106 −0 vendor/a4.io/ssse/pkg/client/client.go
  19. +1 −1 vendor/github.com/aws/aws-sdk-go/aws/corehandlers/user_agent.go
  20. +28 −3 vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go
  21. +1 −1 vendor/github.com/aws/aws-sdk-go/aws/version.go
  22. +16 −0 vendor/github.com/klauspost/reedsolomon/README.md
  23. +1 −0 vendor/github.com/klauspost/reedsolomon/galois_noasm.go
  24. +67 −0 vendor/github.com/klauspost/reedsolomon/galois_ppc64le.go
  25. +126 −0 vendor/github.com/klauspost/reedsolomon/galois_ppc64le.s
  26. +25 −0 vendor/github.com/unrolled/secure/README.md
  27. +1 −0 vendor/github.com/valyala/gozstd/.gitignore
  28. +20 −0 vendor/github.com/valyala/gozstd/.travis.yml
  29. +22 −0 vendor/github.com/valyala/gozstd/LICENSE
  30. +37 −0 vendor/github.com/valyala/gozstd/Makefile
  31. +73 −0 vendor/github.com/valyala/gozstd/README.md
  32. +187 −0 vendor/github.com/valyala/gozstd/dict.go
  33. +1 −0 vendor/github.com/valyala/gozstd/go.mod
  34. +408 −0 vendor/github.com/valyala/gozstd/gozstd.go
  35. BIN vendor/github.com/valyala/gozstd/libzstd_darwin.a
  36. +6 −0 vendor/github.com/valyala/gozstd/libzstd_darwin.go
  37. BIN vendor/github.com/valyala/gozstd/libzstd_linux.a
  38. +6 −0 vendor/github.com/valyala/gozstd/libzstd_linux.go
  39. BIN vendor/github.com/valyala/gozstd/libzstd_windows.a
  40. +6 −0 vendor/github.com/valyala/gozstd/libzstd_windows.go
  41. +237 −0 vendor/github.com/valyala/gozstd/reader.go
  42. +128 −0 vendor/github.com/valyala/gozstd/stream.go
  43. +292 −0 vendor/github.com/valyala/gozstd/writer.go
  44. +267 −0 vendor/github.com/valyala/gozstd/zdict.h
  45. +1,526 −0 vendor/github.com/valyala/gozstd/zstd.h
  46. +92 −0 vendor/github.com/valyala/gozstd/zstd_errors.h
  47. +2 −2 vendor/golang.org/x/sys/unix/README.md
  48. +14 −9 vendor/golang.org/x/sys/unix/mkall.sh
  49. +61 −0 vendor/golang.org/x/sys/unix/mkasm_darwin.go
  50. +12 −6 vendor/golang.org/x/sys/unix/mkerrors.sh
  51. +3 −3 vendor/golang.org/x/sys/unix/mkpost.go
  52. +34 −3 vendor/golang.org/x/sys/unix/mksyscall.go
  53. +7 −0 vendor/golang.org/x/sys/unix/syscall_aix.go
  54. +25 −37 vendor/golang.org/x/sys/unix/syscall_darwin.go
  55. +9 −14 vendor/golang.org/x/sys/unix/syscall_darwin_386.go
  56. +9 −14 vendor/golang.org/x/sys/unix/syscall_darwin_amd64.go
  57. +12 −14 vendor/golang.org/x/sys/unix/syscall_darwin_arm.go
  58. +12 −14 vendor/golang.org/x/sys/unix/syscall_darwin_arm64.go
  59. +31 −0 vendor/golang.org/x/sys/unix/syscall_darwin_libSystem.go
  60. +7 −0 vendor/golang.org/x/sys/unix/syscall_dragonfly.go
  61. +7 −0 vendor/golang.org/x/sys/unix/syscall_freebsd.go
  62. +7 −0 vendor/golang.org/x/sys/unix/syscall_linux.go
  63. +7 −0 vendor/golang.org/x/sys/unix/syscall_netbsd.go
  64. +7 −0 vendor/golang.org/x/sys/unix/syscall_openbsd.go
  65. +7 −0 vendor/golang.org/x/sys/unix/syscall_solaris.go
  66. +0 −7 vendor/golang.org/x/sys/unix/syscall_unix.go
  67. +2,420 −1,776 vendor/golang.org/x/sys/unix/zerrors_linux_sparc64.go
  68. +1,810 −0 vendor/golang.org/x/sys/unix/zsyscall_darwin_386.1_11.go
  69. +955 −219 vendor/golang.org/x/sys/unix/zsyscall_darwin_386.go
  70. +284 −0 vendor/golang.org/x/sys/unix/zsyscall_darwin_386.s
  71. +1,810 −0 vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.1_11.go
  72. +955 −219 vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.go
  73. +284 −0 vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.s
  74. +1,793 −0 vendor/golang.org/x/sys/unix/zsyscall_darwin_arm.1_11.go
  75. +933 −219 vendor/golang.org/x/sys/unix/zsyscall_darwin_arm.go
  76. +282 −0 vendor/golang.org/x/sys/unix/zsyscall_darwin_arm.s
  77. +1,793 −0 vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.1_11.go
  78. +933 −219 vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.go
  79. +282 −0 vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.s
  80. +4 −2 vendor/golang.org/x/sys/unix/zsysnum_linux_sparc64.go
  81. +1 −8 vendor/golang.org/x/sys/unix/ztypes_linux_386.go
  82. +0 −16 vendor/golang.org/x/sys/unix/ztypes_linux_amd64.go
  83. +2 −8 vendor/golang.org/x/sys/unix/ztypes_linux_arm.go
  84. +0 −16 vendor/golang.org/x/sys/unix/ztypes_linux_arm64.go
  85. +1 −6 vendor/golang.org/x/sys/unix/ztypes_linux_mips.go
  86. +0 −16 vendor/golang.org/x/sys/unix/ztypes_linux_mips64.go
  87. +0 −16 vendor/golang.org/x/sys/unix/ztypes_linux_mips64le.go
  88. +1 −6 vendor/golang.org/x/sys/unix/ztypes_linux_mipsle.go
  89. +0 −16 vendor/golang.org/x/sys/unix/ztypes_linux_ppc64.go
  90. +0 −16 vendor/golang.org/x/sys/unix/ztypes_linux_ppc64le.go
  91. +0 −16 vendor/golang.org/x/sys/unix/ztypes_linux_riscv64.go
  92. +1 −20 vendor/golang.org/x/sys/unix/ztypes_linux_s390x.go
  93. +1,389 −98 vendor/golang.org/x/sys/unix/ztypes_linux_sparc64.go
  94. +6 −0 vendor/google.golang.org/appengine/appengine.go
  95. +4 −1 vendor/google.golang.org/appengine/internal/api.go
  96. +9 −3 vendor/google.golang.org/appengine/internal/identity.go
  97. +1 −0 vendor/google.golang.org/appengine/internal/main.go
  98. +7 −0 vendor/google.golang.org/appengine/internal/main_common.go
  99. +21 −0 vendor/google.golang.org/appengine/internal/main_vm.go
  100. +12 −8 vendor/modules.txt
38 go.mod
@@ -1,44 +1,62 @@
module a4.io/blobstash

require (
a4.io/blobsfile v0.0.0-20181029195936-c742249a3522
a4.io/gluapp v0.0.0-20181217122610-c6ba9b02f21b
a4.io/blobsfile v0.1.0
a4.io/gluapp v0.0.0-20181218195258-2be1706b2908
a4.io/gluarequire2 v0.0.0-20170611121149-66e0eb2c6a9f
a4.io/ssse v0.0.0-20181202155639-1949828a8689
bazil.org/fuse v0.0.0-20180421153158-65cc252bf669
github.com/aws/aws-sdk-go v1.16.6
github.com/aws/aws-sdk-go v1.16.11
github.com/blevesearch/segment v0.0.0-20160915185041-762005e7a34f
github.com/carbocation/handlers v0.0.0-20140528190747-c939c6d9ef31 // indirect
github.com/carbocation/interpose v0.0.0-20161206215253-723534742ba3
github.com/cespare/trie v0.0.0-20150610204604-3fe1a95cbba9 // indirect
github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 // indirect
github.com/codegangsta/negroni v1.0.0 // indirect
github.com/cznic/kv v0.0.0-20181122101858-e9cdcade440e
github.com/dave/jennifer v1.2.0
github.com/dustin/go-humanize v1.0.0
github.com/edsrzf/mmap-go v1.0.0 // indirect
github.com/emirpasic/gods v1.12.0 // indirect
github.com/evanphx/json-patch v4.1.0+incompatible
github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab // indirect
github.com/go-stack/stack v1.8.0 // indirect
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db
github.com/goods/httpbuf v0.0.0-20120503183857-5709e9bb814c // indirect
github.com/gorilla/context v1.1.1
github.com/gorilla/handlers v1.4.0
github.com/gorilla/mux v1.6.2
github.com/hashicorp/golang-lru v0.5.0
github.com/inconshreveable/log15 v0.0.0-20180818164646-67afb5ed74ec
github.com/interpose/middleware v0.0.0-20150216143757-05ed56ed52fa // indirect
github.com/justinas/nosurf v0.0.0-20181122113328-3af30e51c05b // indirect
github.com/mattn/go-colorable v0.0.9 // indirect
github.com/mattn/go-isatty v0.0.4 // indirect
github.com/meatballhat/negroni-logrus v0.0.0-20170801195057-31067281800f // indirect
github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
github.com/onsi/ginkgo v1.7.0 // indirect
github.com/onsi/gomega v1.4.3 // indirect
github.com/phyber/negroni-gzip v0.0.0-20180113114010-ef6356a5d029 // indirect
github.com/reiver/go-porterstemmer v1.0.1
github.com/restic/chunker v0.2.0
github.com/robfig/cron v0.0.0-20180505203441-b41be1df6967
github.com/rwcarlsen/goexif v0.0.0-20180518182100-8d986c03457a
github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735 // indirect
github.com/sirupsen/logrus v1.2.0 // indirect
github.com/syndtr/goleveldb v0.0.0-20181128100959-b001fa50d6b2
github.com/unrolled/secure v0.0.0-20181022170031-4b6b7cf51606
github.com/toqueteos/trie v0.0.0-20150530104557-56fed4a05683 // indirect
github.com/unrolled/secure v0.0.0-20181221173256-0d6b5bb13069
github.com/urfave/negroni v1.0.0 // indirect
github.com/vmihailenco/msgpack v4.0.1+incompatible
github.com/xeonx/timeago v1.0.0-rc3
github.com/yuin/gopher-lua v0.0.0-20181214045814-db9ae37725ec
github.com/zpatrick/rbac v0.0.0-20180829190353-d2c4f050cf28
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9
golang.org/x/net v0.0.0-20181217023233-e147a9138326
golang.org/x/sys v0.0.0-20181213200352-4d1cda033e06 // indirect
golang.org/x/net v0.0.0-20181220203305-927f97764cc3
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f // indirect
google.golang.org/appengine v1.4.0 // indirect
gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec
gopkg.in/src-d/enry.v1 v1.6.7
gopkg.in/src-d/go-billy.v4 v4.3.0 // indirect
gopkg.in/src-d/go-git-fixtures.v3 v3.3.0 // indirect
gopkg.in/src-d/go-git.v4 v4.8.1
gopkg.in/toqueteos/substring.v1 v1.0.2 // indirect
gopkg.in/yaml.v2 v2.2.2
)
21 go.sum
@@ -1,12 +1,19 @@
a4.io/blobsfile v0.0.0-20181029195936-c742249a3522 h1:QVL41vUxtKA6ceXG4W9ha6ZOERVmpjGRSBW9bbuaje4=
a4.io/blobsfile v0.0.0-20181029195936-c742249a3522/go.mod h1:jTrsc9CgnEavpl6Tmowi2bZbGXldVGr5gvkFsS12bKs=
a4.io/blobsfile v0.1.0 h1:nE9v20zzfL7UKvZuA2J42ncYBCqZ9mgVU9am9+XZdZk=
a4.io/blobsfile v0.1.0/go.mod h1:kJFL3M8OxlvHZWzxZ6C9o+ky9iJHmx0jZj59nilWzJM=
a4.io/blobstash v0.0.0-20181216235946-aa2d4a59f200/go.mod h1:PVI3EM/VmUQAz7pbz/govGO4gHypTF5YWhS56qETj+M=
a4.io/blobstash v0.0.0-20181218201750-765e41187e8a/go.mod h1:QH1JUxPtdWiC/hCXrfzS03p5tX9mqALuBzUd2yYflso=
a4.io/gluapp v0.0.0-20181203183836-c136dc4e9123 h1:k15yXbQ2YfmsxIvcVmuNEGNaE4Gh792Ciye6nTDV0rM=
a4.io/gluapp v0.0.0-20181203183836-c136dc4e9123/go.mod h1:rK/CQwI+tDICKCR1szNtBP0rJdH1LCrO/ZnculcIjWI=
a4.io/gluapp v0.0.0-20181217122610-c6ba9b02f21b h1:0mDg1XGqGMCUMIu69JHmJp17iAKpA8549I26/knLx20=
a4.io/gluapp v0.0.0-20181217122610-c6ba9b02f21b/go.mod h1:hDz8O30eiYv+1bAFzssTvbRaLy27xwk7pdR7v2md7Ew=
a4.io/gluapp v0.0.0-20181218195258-2be1706b2908 h1:4X4w3ef5+gyUErHpxdyMoHXSKUCY9naICJGwdwceLc4=
a4.io/gluapp v0.0.0-20181218195258-2be1706b2908/go.mod h1:hDz8O30eiYv+1bAFzssTvbRaLy27xwk7pdR7v2md7Ew=
a4.io/gluarequire2 v0.0.0-20170611121149-66e0eb2c6a9f h1:mfEWN0Dd2AfIXU5WO5ZfqbFVk63Qz5M/CANs182pm+U=
a4.io/gluarequire2 v0.0.0-20170611121149-66e0eb2c6a9f/go.mod h1:t7OhwCmPQfuUf8cjm7n8chSbZt5CTILu+dTLu1MQKjQ=
a4.io/ssse v0.0.0-20181202155639-1949828a8689 h1:xXG2KAzqyucPIo7c7zEW9aGUDXV+Fqg5jVrQiw7s2DU=
a4.io/ssse v0.0.0-20181202155639-1949828a8689/go.mod h1:/4k4qDJv4lDmiIcMs9k/5Rs7bU/1FkIvu42oMyf5A7Y=
bazil.org/fuse v0.0.0-20180421153158-65cc252bf669 h1:FNCRpXiquG1aoyqcIWVFmpTSKVcx2bQD38uZZeGtdlw=
bazil.org/fuse v0.0.0-20180421153158-65cc252bf669/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8=
github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs=
@@ -15,6 +22,8 @@ github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
github.com/aws/aws-sdk-go v1.16.6 h1:pig/KdfESvIv4gUu1B8nVAJAURxbPCTt6e5u79Nqxqc=
github.com/aws/aws-sdk-go v1.16.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/aws/aws-sdk-go v1.16.11 h1:g/c7gJeVyHoXCxM2fddS85bPGVkBF8s2q8t3fyElegc=
github.com/aws/aws-sdk-go v1.16.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/blevesearch/segment v0.0.0-20160915185041-762005e7a34f h1:kqbi9lqXLLs+zfWlgo1PIiRQ86n33K1JKotjj4rSYOg=
github.com/blevesearch/segment v0.0.0-20160915185041-762005e7a34f/go.mod h1:IInt5XRvpiGE09KOk9mmCMLjHhydIhNPKPPFLFBB7L8=
github.com/carbocation/handlers v0.0.0-20140528190747-c939c6d9ef31 h1:SDMgCFII5drFRIyAaihze9ceRMpTt1FW6Q5jjpc2u4c=
@@ -105,6 +114,8 @@ github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE
github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/reedsolomon v1.7.0 h1:pLFmRKGko2ZieiTGyo9DahLCIuljyxm+Zzhz/fYEonE=
github.com/klauspost/reedsolomon v1.7.0/go.mod h1:CwCi+NUr9pqSVktrkN+Ondf06rkhYZ/pcNv7fu+8Un4=
github.com/klauspost/reedsolomon v1.8.0 h1:lvvOkvk64cE1EGbBIgFk7WSOOsI1GexpuLiT7zjab6g=
github.com/klauspost/reedsolomon v1.8.0/go.mod h1:CwCi+NUr9pqSVktrkN+Ondf06rkhYZ/pcNv7fu+8Un4=
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@@ -165,8 +176,12 @@ github.com/toqueteos/trie v0.0.0-20150530104557-56fed4a05683 h1:ej8ns+4aeQO+mm9V
github.com/toqueteos/trie v0.0.0-20150530104557-56fed4a05683/go.mod h1:Ywk48QhEqhU1+DwhMkJ2x7eeGxDHiGkAdc9+0DYcbsM=
github.com/unrolled/secure v0.0.0-20181022170031-4b6b7cf51606 h1:dU9yXzNi9rl6Mou7+3npdfPyeFPb2+7BHs3zL47bhPY=
github.com/unrolled/secure v0.0.0-20181022170031-4b6b7cf51606/go.mod h1:mnPT77IAdsi/kV7+Es7y+pXALeV3h7G6dQF6mNYjcLA=
github.com/unrolled/secure v0.0.0-20181221173256-0d6b5bb13069 h1:RKeYksgIwGE8zFJTvXI1WWx09QPrGyaVFMy0vpU7j/o=
github.com/unrolled/secure v0.0.0-20181221173256-0d6b5bb13069/go.mod h1:mnPT77IAdsi/kV7+Es7y+pXALeV3h7G6dQF6mNYjcLA=
github.com/urfave/negroni v1.0.0 h1:kIimOitoypq34K7TG7DUaJ9kq/N4Ofuwi1sjz0KipXc=
github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4=
github.com/valyala/gozstd v1.2.1 h1:ZZcVQLO6Ff5I3Ca6OMZFmg5SA9lan3C7kIS84YlRjpY=
github.com/valyala/gozstd v1.2.1/go.mod h1:oYOS+oJovjw9ewtrwEYb9+ybolEXd6pHyLMuAWN5zts=
github.com/vmihailenco/msgpack v4.0.1+incompatible h1:RMF1enSPeKTlXrXdOcqjFUElywVZjjC6pqse21bKbEU=
github.com/vmihailenco/msgpack v4.0.1+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=
github.com/xanzy/ssh-agent v0.2.0 h1:Adglfbi5p9Z0BmK2oKU9nTG+zKfniSfnaMYB+ULd+Ro=
@@ -186,6 +201,8 @@ golang.org/x/net v0.0.0-20181213202711-891ebc4b82d6 h1:gT0Y6H7hbVPUtvtk0YGxMXPgN
golang.org/x/net v0.0.0-20181213202711-891ebc4b82d6/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181217023233-e147a9138326 h1:iCzOf0xz39Tstp+Tu/WwyGjUXCk34QhQORRxBeXXTA4=
golang.org/x/net v0.0.0-20181217023233-e147a9138326/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181220203305-927f97764cc3 h1:eH6Eip3UpmR+yM/qI9Ijluzb1bNv/cAU/n+6l8tRSis=
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -197,10 +214,14 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e h1:o3PsSEY8E4eXWkXrIP9YJALUk
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181213200352-4d1cda033e06 h1:0oC8rFnE+74kEmuHZ46F6KHsMr5Gx2gUQPuNz28iQZM=
golang.org/x/sys v0.0.0-20181213200352-4d1cda033e06/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6 h1:IcgEB62HYgAhX0Nd/QrVgZlxlcyxbGQHElLUhW2X4Fo=
golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
google.golang.org/appengine v1.3.0 h1:FBSsiFRMz3LBeXIomRnVzrQwSDj4ibvcRexLG0LZGQk=
google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -45,6 +45,7 @@

logging.info('Enumerating blobs')
blobs_resp = c._get('/api/blobstore/blobs').json()
print(blobs_resp)
assert len(blobs_resp['refs']) == 1, 'failed to enumate blobs, expected 1 got {}'.format(len(blobs_resp['refs']))
blob_ref = blobs_resp['refs'][0]
assert blob_ref['hash'] == blob2.hash, 'failed to enumate blobs, hash does not match, expected {} got {}'.format(
@@ -115,20 +115,6 @@ func (bs *BlobStoreAPI) blobHandler() func(http.ResponseWriter, *http.Request) {
}
// FIXME(tsileo): clean this case, skip a decoding/encoding round and return the bytes as is from the
// backend storage
if r.Header.Get("Accept-Encoding") == "snappy" {
blob, err := bs.bs.GetEncoded(ctx, vars["hash"])
if err != nil {
if err == blobsfile.ErrBlobNotFound {
httputil.WriteJSONError(w, http.StatusNotFound, http.StatusText(http.StatusNotFound))
} else {
httputil.Error(w, err)
}
return
}
httputil.WriteEncoded(r, w, blob)
return
}

blob, err := bs.bs.Get(ctx, vars["hash"])
if err != nil {
if err == blobsfile.ErrBlobNotFound {
@@ -7,7 +7,6 @@ import (
"fmt"
"path/filepath"

"github.com/golang/snappy"
log "github.com/inconshreveable/log15"

"a4.io/blobsfile"
@@ -62,8 +61,7 @@ type BlobStore struct {

func New(logger log.Logger, root bool, dir string, conf2 *config.Config, hub *hub.Hub) (*BlobStore, error) {
logger.Debug("init")

back, err := blobsfile.New(&blobsfile.Opts{Directory: filepath.Join(dir, "blobs")})
back, err := blobsfile.New(&blobsfile.Opts{Compression: blobsfile.Zstandard, Directory: filepath.Join(dir, "blobs")})
if err != nil {
return nil, fmt.Errorf("failed to init BlobsFile: %v", err)
}
@@ -181,75 +179,6 @@ func (bs *BlobStore) Put(ctx context.Context, blob *blob.Blob) error {
return nil
}

func (bs *BlobStore) GetEncoded(ctx context.Context, hash string) ([]byte, error) {
bs.log.Info("OP Get (encoded)", "hash", hash)
blob, err := bs.back.GetEncoded(hash)
switch err {
case nil:
case blobsfile.ErrBlobNotFound:
if !bs.root || bs.s3back == nil {
return nil, err
}

// The blob may be queued for download
inFlight, blb, berr := bs.s3back.BlobWaitingForDownload(hash)
if berr != nil {
return nil, berr
}
if inFlight {
// The blob is queued for download, force the download
blob = blb.Data
blob = snappy.Encode(nil, blb.Data)
err = nil
break
}

// If there's a data cache, it means the blob must be stored on S3
if bs.dataCache != nil {
cached, err := bs.dataCache.Stat(hash)
if err != nil {
return nil, err
}
if cached {
bs.log.Debug("blob found in the data cache", "hash", hash)
blob, _, err = bs.dataCache.Get(hash)
blob = snappy.Encode(nil, blob)
err = nil
break
}

// If the blob is available on S3, download it and add it to the data cache
indexed, err := bs.s3back.Indexed(hash)
if err != nil {
return nil, err
}

if indexed {
bs.log.Debug("blob found on S3", "hash", hash)
blob, err = bs.s3back.Get(hash)
if err != nil {
return nil, err
}
if cerr := bs.dataCache.Add(hash, blob); cerr != nil {
return nil, cerr
}
blob = snappy.Encode(nil, blob)
break
}
}

// Return the original error
return nil, err
default:
return nil, err
}

readCountVar.Add(1)
readVar.Add(int64(len(blob)))

return blob, nil
}

func (bs *BlobStore) Get(ctx context.Context, hash string) ([]byte, error) {
bs.log.Info("OP Get", "hash", hash)
blob, err := bs.back.Get(hash)
@@ -16,7 +16,7 @@ import (
)

var (
NodeBlobHeader = []byte("#blob/node\n")
NodeBlobHeader = []byte("#blobstash/node\n")
NodeBlobMsgpackEncoding = byte('1')
NodeBlobOverhead = len(NodeBlobHeader) + 1
)
@@ -187,9 +187,9 @@ func (up *Uploader) PutDir(path string) (*rnode.RawNode, error) {
up.DirExplorer(path, n, nodes)
defer close(nodes)
}()
// Upload discovered files (100 file descriptor at the same time max).
// Upload discovered files (5 file descriptor at the same time max).
wg.Add(1)
l := make(chan struct{}, 25)
l := make(chan struct{}, 5)
go func() {
defer wg.Done()
for f := range nodes {
@@ -566,7 +566,7 @@ func (gs *GitServer) nsHandler(w http.ResponseWriter, r *http.Request) {
}

type GitRepoRefs struct {
Branches []*RefSummary `json"branches"`
Branches []*RefSummary `json:"branches"`
Tags []*RefSummary `json:"tags"`
}

@@ -197,8 +197,9 @@ func newFileStat(L *lua.LState, name string, additions, deletions int) *lua.LTab
}

func convertRefSummary(L *lua.LState, refSummary *gitserver.RefSummary) *lua.LTable {
tbl := L.CreateTable(0, 6)
tbl := L.CreateTable(0, 7)
tbl.RawSetH(lua.LString("commit_time_ago"), lua.LString(timeago.English.Format(mustParse(refSummary.Commit.Committer.Date))))
tbl.RawSetH(lua.LString("commit_short_hash"), lua.LString(refSummary.Commit.Hash[:8]))
tbl.RawSetH(lua.LString("commit_hash"), lua.LString(refSummary.Commit.Hash))
tbl.RawSetH(lua.LString("commit_message"), lua.LString(refSummary.Commit.Message))
tbl.RawSetH(lua.LString("commit_author_name"), lua.LString(refSummary.Commit.Committer.Name))
@@ -340,14 +340,6 @@ func (bs *BlobStore) Get(ctx context.Context, hash string) ([]byte, error) {

}

func (bs *BlobStore) GetEncoded(ctx context.Context, hash string) ([]byte, error) {
dataContext, err := bs.s.dataContext(ctx)
if err != nil {
return nil, err
}
return dataContext.BlobStoreProxy().GetEncoded(ctx, hash)
}

func (bs *BlobStore) Stat(ctx context.Context, hash string) (bool, error) {
dataContext, err := bs.s.dataContext(ctx)
if err != nil {
@@ -301,7 +301,6 @@ func (p *KvStoreProxy) Keys(ctx context.Context, start, end string, limit int) (
type BlobStore interface {
Put(ctx context.Context, blob *blob.Blob) error
Get(ctx context.Context, hash string) ([]byte, error)
GetEncoded(ctx context.Context, hash string) ([]byte, error)
Stat(ctx context.Context, hash string) (bool, error)
Enumerate(ctx context.Context, start, end string, limit int) ([]*blob.SizedBlobRef, string, error)
Close() error
@@ -324,18 +323,6 @@ func (p *BlobStoreProxy) Get(ctx context.Context, hash string) ([]byte, error) {
return data, nil
}

func (p *BlobStoreProxy) GetEncoded(ctx context.Context, hash string) ([]byte, error) {
data, err := p.BlobStore.GetEncoded(ctx, hash)
switch err {
case nil:
case blobsfile.ErrBlobNotFound:
return p.ReadSrc.GetEncoded(ctx, hash)
default:
return nil, err
}
return data, nil
}

func (p *BlobStoreProxy) Stat(ctx context.Context, hash string) (bool, error) {
exists, err := p.BlobStore.Stat(ctx, hash)
if err != nil {

Some generated files are not rendered by default. Learn more.

0 comments on commit 1fa5baf

Please sign in to comment.
You can’t perform that action at this time.