Skip to content

Commit

Permalink
fixed Makefile and preparation for release
Browse files Browse the repository at this point in the history
  • Loading branch information
10d9e committed Jun 5, 2023
1 parent 659c195 commit 347a8b7
Show file tree
Hide file tree
Showing 4 changed files with 75 additions and 7 deletions.
4 changes: 2 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
build:
go build -o fastcommp
go build -o fastcommp cmd/main.go

run:
go run main.go
go run cmd/main.go

clean:
rm ./fastcommp 8G-payload.bin
Expand Down
12 changes: 12 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,18 @@

Filecoin fast piece commitment summation tool.

# usage

```go
fast := new(fastcommp.CommpWriter)
fast.Write(data)
sum, err := fast.Sum()
if err != nil {
panic(err)
}
fmt.Printf("commP: %s\n", sum.PieceCID.String())
```

# build

`make build`
Expand Down
50 changes: 50 additions & 0 deletions cmd/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
package main

import (
"encoding/json"
"fmt"
"io/ioutil"
"os"
"time"

"github.com/application-research/fastcommp"
)

func main() {
// Get the file name from the command-line arguments
if len(os.Args) != 2 {
fmt.Printf("Usage: %s <filename>\n", os.Args[0])
return
}
fileName := os.Args[1]

start := time.Now()
data, err := ioutil.ReadFile(fileName)
if err != nil {
fmt.Println("Error reading file:", err)
return
}

elapsed := time.Since(start)
fmt.Printf("Elapsed file read time: %s\n", elapsed)

fast := new(fastcommp.CommpWriter)
start = time.Now()
fast.Write(data)
sum, err := fast.Sum()
if err != nil {
panic(err)
}

elapsed = time.Since(start)
fmt.Printf("Elapsed commP time: %s\n", elapsed)
fmt.Printf("commP: %s\n", sum.PieceCID.String())

// Convert the sum results to a JSON string
results, err := json.MarshalIndent(sum, "", " ")
if err != nil {
panic(err)
}
fmt.Println(string(results))

}
16 changes: 11 additions & 5 deletions fastcommp.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,16 +27,16 @@ type DataCIDSize struct {
const commPBufPad = abi.PaddedPieceSize(8 << 20)

// CommPBuf is the size of the buffer used to calculate commP
const CommPBuf = abi.UnpaddedPieceSize(commPBufPad - (commPBufPad / 128)) // can't use .Unpadded() for const
const CommPBuf = abi.UnpaddedPieceSize(commPBufPad - (commPBufPad / 128))

// ciderr is a cid and an error
type ciderr struct {
c cid.Cid
err error
}

// DataCidWriter is a writer that calculates the CommP
type DataCidWriter struct {
// CommpWriter is a writer that calculates the CommP
type CommpWriter struct {
len int64
buf [CommPBuf]byte
leaves []chan ciderr
Expand All @@ -46,7 +46,7 @@ type DataCidWriter struct {
}

// Write writes data to the DataCidWriter
func (w *DataCidWriter) Write(p []byte) (int, error) {
func (w *CommpWriter) Write(p []byte) (int, error) {
if w.throttle == nil {
w.throttle = make(chan int, runtime.NumCPU())
}
Expand All @@ -57,6 +57,7 @@ func (w *DataCidWriter) Write(p []byte) (int, error) {
w.tbufs = make([][CommPBuf]byte, cap(w.throttle))
}

// process last non-zero leaf if exists and we have data to write
n := len(p)
for len(p) > 0 {
buffered := int(w.len % int64(len(w.buf)))
Expand All @@ -69,16 +70,19 @@ func (w *DataCidWriter) Write(p []byte) (int, error) {
p = p[copied:]
w.len += int64(copied)

// if we filled the buffer, process it
if copied > 0 && w.len%int64(len(w.buf)) == 0 {
leaf := make(chan ciderr, 1)
bufIdx := <-w.throttle
copy(w.tbufs[bufIdx][:], w.buf[:])

// process leaf in a goroutine
go func() {
defer func() {
w.throttle <- bufIdx
}()

// calculate commP for this leaf and send it to the channel
cc := new(commp.Calc)
_, _ = cc.Write(w.tbufs[bufIdx][:])
p, _, _ := cc.Digest()
Expand All @@ -89,17 +93,19 @@ func (w *DataCidWriter) Write(p []byte) (int, error) {
}
}()

// add leaf to list
w.leaves = append(w.leaves, leaf)
}
}
return n, nil
}

func (w *DataCidWriter) Sum() (DataCIDSize, error) {
func (w *CommpWriter) Sum() (DataCIDSize, error) {
// process last non-zero leaf if exists
lastLen := w.len % int64(len(w.buf))
rawLen := w.len

// wait for all leaves to finish
leaves := make([]cid.Cid, len(w.leaves))
for i, leaf := range w.leaves {
r := <-leaf
Expand Down

0 comments on commit 347a8b7

Please sign in to comment.