Skip to content

Commit

Permalink
blake3 support
Browse files Browse the repository at this point in the history
  • Loading branch information
@boyter
boyter committed Nov 14, 2021
1 parent f6fd85a commit afe1ab6
Show file tree
Hide file tree
Showing 325 changed files with 5,288 additions and 221,890 deletions.
3 changes: 2 additions & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,12 @@ go 1.17
require (
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1
github.com/spf13/cobra v0.0.3
github.com/zeebo/blake3 v0.2.1
golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f
)

require (
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/spf13/pflag v1.0.3 // indirect
golang.org/x/sys v0.0.0-20190516110030-61b9204099cb // indirect
golang.org/x/sys v0.0.0-20201014080544-cc95f250f6bc // indirect
)
4 changes: 4 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8=
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/zeebo/blake3 v0.2.1 h1:O+N0Y8Re2XAYjp0adlZDA2juyRguhMfPCgh8YIf7vyE=
github.com/zeebo/blake3 v0.2.1/go.mod h1:TSQ0KjMH+pht+bRyvVooJ1rBpvvngSGaPISafq9MxJk=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f h1:R423Cnkcp5JABoeemiGEPlt9tHXFfw5kvc0yqlxRPWo=
golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
Expand All @@ -14,4 +16,6 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190516110030-61b9204099cb h1:k07iPOt0d6nEnwXF+kHB+iEg+WSuKe/SOQuFM2QoD+E=
golang.org/x/sys v0.0.0-20190516110030-61b9204099cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201014080544-cc95f250f6bc h1:HVFDs9bKvTxP6bh1Rj9MCSo+UmafQtI8ZWDPVwVk9g4=
golang.org/x/sys v0.0.0-20201014080544-cc95f250f6bc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
7 changes: 7 additions & 0 deletions processor/formatters.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,9 @@ func toSum(input chan Result) string {
if hasHash(HashNames.Blake2b512) {
str.WriteString(res.Blake2b512 + " " + res.File + "\n")
}
if hasHash(HashNames.Blake3) {
str.WriteString(res.Blake3 + " " + res.File + "\n")
}
if hasHash(HashNames.Sha3224) {
str.WriteString(res.Sha3224 + " " + res.File + "\n")
}
Expand Down Expand Up @@ -155,6 +158,9 @@ func toText(input chan Result) (string, bool) {
if hasHash(HashNames.Blake2b512) {
str.WriteString("Blake2b-512 " + res.Blake2b512 + "\n")
}
if hasHash(HashNames.Blake3) {
str.WriteString(" Blake3 " + res.Blake3 + "\n")
}
if hasHash(HashNames.Sha3224) {
str.WriteString(" SHA3-224 " + res.Sha3224 + "\n")
}
Expand Down Expand Up @@ -330,6 +336,7 @@ func printHashes() {
fmt.Println(fmt.Sprintf(" SHA512 (%s)", HashNames.SHA512))
fmt.Println(fmt.Sprintf("Blake2b-256 (%s)", HashNames.Blake2b256))
fmt.Println(fmt.Sprintf("Blake2b-512 (%s)", HashNames.Blake2b512))
fmt.Println(fmt.Sprintf(" Blake3 (%s)", HashNames.Blake3))
fmt.Println(fmt.Sprintf(" SHA3-224 (%s)", HashNames.Sha3224))
fmt.Println(fmt.Sprintf(" SHA3-256 (%s)", HashNames.Sha3256))
fmt.Println(fmt.Sprintf(" SHA3-384 (%s)", HashNames.Sha3384))
Expand Down
3 changes: 2 additions & 1 deletion processor/processor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ var isDir = false
var FileListQueueSize = 1000

// Number of bytes in a size to enable memory maps or streaming
var StreamSize int64 = 1000000
var StreamSize int64 = 1_000_000

// If set will enable the internal file audit logic to kick in
var FileAudit = false
Expand All @@ -70,6 +70,7 @@ var HashNames = Result{
SHA512: "sha512",
Blake2b256: "blake2b256",
Blake2b512: "blake2b512",
Blake3: "blake3",
Sha3224: "sha3224",
Sha3256: "sha3256",
Sha3384: "sha3384",
Expand Down
1 change: 1 addition & 0 deletions processor/structs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ type Result struct {
SHA512 string
Blake2b256 string
Blake2b512 string
Blake3 string
Sha3224 string
Sha3256 string
Sha3384 string
Expand Down
48 changes: 47 additions & 1 deletion processor/workers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"encoding/hex"
"fmt"
"github.com/minio/blake2b-simd"
"github.com/zeebo/blake3"
"golang.org/x/crypto/md4"
"golang.org/x/crypto/sha3"
"io"
Expand Down Expand Up @@ -112,6 +113,7 @@ func processScanner(filename string) (Result, error) {
sha512_d := sha512.New()
blake2b_256_d := blake2b.New256()
blake2b_512_d := blake2b.New512()
blake3_d := blake3.New()
sha3_224_d := sha3.New224()
sha3_256_d := sha3.New256()
sha3_384_d := sha3.New384()
Expand All @@ -124,6 +126,7 @@ func processScanner(filename string) (Result, error) {
sha512c := make(chan []byte, 10)
blake2b_256_c := make(chan []byte, 10)
blake2b_512_c := make(chan []byte, 10)
blake3c := make(chan []byte, 10)
sha3_224_c := make(chan []byte, 10)
sha3_256_c := make(chan []byte, 10)
sha3_384_c := make(chan []byte, 10)
Expand Down Expand Up @@ -201,6 +204,16 @@ func processScanner(filename string) (Result, error) {
}()
}

if hasHash(HashNames.Blake3) {
wg.Add(1)
go func() {
for b := range blake3c {
blake3_d.Write(b)
}
wg.Done()
}()
}

if hasHash(HashNames.Sha3224) {
wg.Add(1)
go func() {
Expand Down Expand Up @@ -238,7 +251,7 @@ func processScanner(filename string) (Result, error) {
}()
}

data := make([]byte, 4194304)
data := make([]byte, 4_194_304)
for {
n, err := file.Read(data)
if err != nil && err != io.EOF {
Expand Down Expand Up @@ -272,6 +285,9 @@ func processScanner(filename string) (Result, error) {
if hasHash(HashNames.Blake2b512) {
blake2b_512_c <- tmp[:n]
}
if hasHash(HashNames.Blake3) {
blake3c <- tmp[:n]
}
if hasHash(HashNames.Sha3224) {
sha3_224_c <- tmp[:n]
}
Expand All @@ -297,6 +313,7 @@ func processScanner(filename string) (Result, error) {
close(sha512c)
close(blake2b_256_c)
close(blake2b_512_c)
close(blake3c)
close(sha3_224_c)
close(sha3_256_c)
close(sha3_384_c)
Expand All @@ -314,6 +331,7 @@ func processScanner(filename string) (Result, error) {
SHA512: hex.EncodeToString(sha512_d.Sum(nil)),
Blake2b256: hex.EncodeToString(blake2b_256_d.Sum(nil)),
Blake2b512: hex.EncodeToString(blake2b_512_d.Sum(nil)),
Blake3: hex.EncodeToString(blake3_d.Sum(nil)),
Sha3224: hex.EncodeToString(sha3_224_d.Sum(nil)),
Sha3256: hex.EncodeToString(sha3_256_d.Sum(nil)),
Sha3384: hex.EncodeToString(sha3_384_d.Sum(nil)),
Expand All @@ -333,6 +351,7 @@ func processStandardInput(output chan Result) {
sha512_d := sha512.New()
blake2b_256_d := blake2b.New256()
blake2b_512_d := blake2b.New512()
blake3_d := blake3.New()
sha3_224_d := sha3.New224()
sha3_256_d := sha3.New256()
sha3_384_d := sha3.New384()
Expand All @@ -345,6 +364,7 @@ func processStandardInput(output chan Result) {
sha512c := make(chan []byte, 10)
blake2b_256_c := make(chan []byte, 10)
blake2b_512_c := make(chan []byte, 10)
blake3c := make(chan []byte, 10)
sha3_224_c := make(chan []byte, 10)
sha3_256_c := make(chan []byte, 10)
sha3_384_c := make(chan []byte, 10)
Expand Down Expand Up @@ -422,6 +442,16 @@ func processStandardInput(output chan Result) {
}()
}

if hasHash(HashNames.Blake3) {
wg.Add(1)
go func() {
for b := range blake3c {
blake3_d.Write(b)
}
wg.Done()
}()
}

if hasHash(HashNames.Sha3224) {
wg.Add(1)
go func() {
Expand Down Expand Up @@ -499,6 +529,9 @@ func processStandardInput(output chan Result) {
if hasHash(HashNames.Blake2b512) {
blake2b_512_c <- buf
}
if hasHash(HashNames.Blake3) {
blake3c <- buf
}
if hasHash(HashNames.Sha3224) {
sha3_224_c <- buf
}
Expand All @@ -524,6 +557,7 @@ func processStandardInput(output chan Result) {
close(sha512c)
close(blake2b_256_c)
close(blake2b_512_c)
close(blake3c)
close(sha3_224_c)
close(sha3_256_c)
close(sha3_384_c)
Expand All @@ -541,6 +575,7 @@ func processStandardInput(output chan Result) {
SHA512: hex.EncodeToString(sha512_d.Sum(nil)),
Blake2b256: hex.EncodeToString(blake2b_256_d.Sum(nil)),
Blake2b512: hex.EncodeToString(blake2b_512_d.Sum(nil)),
Blake3: hex.EncodeToString(blake3_d.Sum(nil)),
Sha3224: hex.EncodeToString(sha3_224_d.Sum(nil)),
Sha3256: hex.EncodeToString(sha3_256_d.Sum(nil)),
Sha3384: hex.EncodeToString(sha3_384_d.Sum(nil)),
Expand Down Expand Up @@ -811,6 +846,17 @@ func processReadFile(filename string, content *[]byte) (Result, error) {
}
}

if hasHash(HashNames.Blake3) {
startTime = makeTimestampNano()
d := blake3.New()
d.Write(*content)
result.Blake3 = hex.EncodeToString(d.Sum(nil))

if Trace {
printTrace(fmt.Sprintf("nanoseconds processing blake3: %s: %d", filename, makeTimestampNano()-startTime))
}
}

if hasHash(HashNames.Sha3224) {
startTime = makeTimestampNano()
d := sha3.New224()
Expand Down
6 changes: 6 additions & 0 deletions vendor/github.com/zeebo/blake3/.gitignore
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

125 changes: 125 additions & 0 deletions vendor/github.com/zeebo/blake3/LICENSE
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

11 changes: 11 additions & 0 deletions vendor/github.com/zeebo/blake3/Makefile
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit afe1ab6

Please sign in to comment.