Skip to content
Permalink
Browse files

Implement fsck (#46)

This fsck travels into all directories in a filesystem dcrtime database
and ensures that all hashes and timestamps are consistent. It also
verifies that flushed records are in fact stored on the blockchain and
verifies the merkle roots in the flush record and reality.

It currently fixes data errors caused by three bugs. These bugs have
since been fixed but the data incoherence was never corrected.

The fsck has a journaling function which records all actions taken. This
is useful in case at a later time there needs to be a correction.
  • Loading branch information...
marcopeereboom committed Feb 22, 2019
1 parent 81e691f commit d496e2f143ecbb19868865364ed4d5a2064219b8
@@ -43,7 +43,9 @@ The dcrtime stack as as follows:

## Library and interfaces
* api/v1 - JSON REST API for dcrtime clients.
* cmd/dcrtime - Client reference implementation
* cmd/dcrtime - Client reference implementation.
* cmd/dcrtime_dump - Data dump/restore tool for filesystem based backend.
* cmd/dcrtime_fsck - Data integrity tool for filesystem based backend.
* cmd/dcrtime_unflush - Debug backend tool to either delete the flush record or reset the chain timestamp.
* cmd/dcrtime_timestamp - Tool to convert between various timestamp formats.
* merkle - Merkle algorithm implementation.
@@ -6,17 +6,19 @@ import (
"os"
"path/filepath"

"github.com/decred/dcrd/chaincfg"
"github.com/decred/dcrd/dcrutil"
"github.com/decred/dcrtime/dcrtimed/backend/filesystem"
)

var (
defaultHomeDir = dcrutil.AppDataDir("dcrtimed", false)
testnet = flag.Bool("testnet", false, "Use testnet port")
dumpJSON = flag.Bool("json", false, "Dump JSON")
restore = flag.Bool("restore", false, "Restore backend, -destination is required")
destination = flag.String("destination", "", "Restore destination")
fsRoot = flag.String("source", defaultHomeDir, "Source directory")

destination = flag.String("destination", "", "Restore destination")
dumpJSON = flag.Bool("json", false, "Dump JSON")
restore = flag.Bool("restore", false, "Restore backend, -destination is required")
fsRoot = flag.String("source", "", "Source directory")
testnet = flag.Bool("testnet", false, "Use testnet port")
)

func _main() error {
@@ -26,10 +28,7 @@ func _main() error {
if *destination == "" {
return fmt.Errorf("-destination must be set")
}
}

if *restore {
// Restore
fs, err := filesystem.NewRestore(*destination)
if err != nil {
return err
@@ -39,14 +38,20 @@ func _main() error {
return fs.Restore(os.Stdin, true, *destination)
}

// Dump
var root string
if *testnet {
root = filepath.Join(*fsRoot, "data", "testnet2")
} else {
root = filepath.Join(*fsRoot, "data", "mainnet")
root := *fsRoot
if root == "" {
root = filepath.Join(defaultHomeDir, "data")
if *testnet {
root = filepath.Join(root, chaincfg.TestNet3Params.Name)
} else {
root = filepath.Join(root, chaincfg.MainNetParams.Name)
}
}

// Dump

fmt.Printf("=== Root: %v\n", root)

fs, err := filesystem.NewDump(root)
if err != nil {
return err
@@ -0,0 +1,46 @@
dcrtime_fsck
============

The filesystem backend can under rare circumstances become incoherent. This
tool iterates over all timestamp directories and corrects known failures.

## Flags

```
-file Journal file. When set actions that will/would be taken are
journaled. This flag works independently of the -fix flag.
-fix Attempt to correct encountered failures.
-host Non default block explorer host. Defaults based on -testnet
flag.
-printhashes Print all hashes encountered during the run. This is very
loud.
-source Non default source directory of the filesystem backend.
-testnet Use testnet.
-v Verbose
```

## Important

Note that the journal may not be identical between a dry- and real run. This
can happen as the filesystem is modified and thus can affect the result of the
journal. This is normal.

The filesystem backend uses lazy timestamp record flushes in order to keep the
source code as simple as possible. This has a result that unless a user has
requested the timestamp information for a given unflushed hash the entire flush
record does not exist. In the `dcrtime_fsck` tool that manifests as `Unflushed`
hash prints. This is normal.

## Examples

Run fsck non-verbose, use non-default filesystem source path and output
potential corrections to `journal.json`.
```
$ dcrtime_fsck -file journal.json -source ~/dcrtime/data/mainnet/
=== Root: /home/marco/dcrtime/data/mainnet/
=== FSCK started Mon Feb 18 14:41:08 CST 2019
--- Phase 1: checking timestamp directories
--- Phase 2: checking global timestamp database
--- Phase 3: checking duplicate digests
=== FSCK completed Mon Feb 18 14:42:50 CST 2019
```
@@ -0,0 +1,76 @@
package main

import (
"flag"
"fmt"
"os"
"path/filepath"
"strings"

"github.com/decred/dcrd/chaincfg"
"github.com/decred/dcrd/dcrutil"
"github.com/decred/dcrtime/dcrtimed/backend"
"github.com/decred/dcrtime/dcrtimed/backend/filesystem"
)

var (
defaultHomeDir = dcrutil.AppDataDir("dcrtimed", false)

file = flag.String("file", "", "journal of modifications if used (will be written despite -fix)")
fix = flag.Bool("fix", false, "Try to correct correctable failures")
dcrdataHost = flag.String("host", "", "dcrdata block explorer")
printHashes = flag.Bool("printhashes", false, "Print all hashes")
fsRoot = flag.String("source", "", "Source directory")
testnet = flag.Bool("testnet", false, "Use testnet port")
verbose = flag.Bool("v", false, "Print more information during run")
)

func _main() error {
flag.Parse()

root := *fsRoot
if root == "" {
root = filepath.Join(defaultHomeDir, "data")
if *testnet {
root = filepath.Join(root, chaincfg.TestNet3Params.Name)
} else {
root = filepath.Join(root, chaincfg.MainNetParams.Name)
}
}

if *dcrdataHost == "" {
if *testnet {
*dcrdataHost = "https://testnet.dcrdata.org/api/tx/"
} else {
*dcrdataHost = "https://explorer.dcrdata.org/api/tx/"
}
} else {
if !strings.HasSuffix(*dcrdataHost, "/") {
*dcrdataHost += "/"
}
}

fmt.Printf("=== Root: %v\n", root)

fs, err := filesystem.NewDump(root)
if err != nil {
return err
}
defer fs.Close()

return fs.Fsck(&backend.FsckOptions{
Verbose: *verbose,
PrintHashes: *printHashes,
Fix: *fix,
URL: *dcrdataHost,
File: *file,
})
}

func main() {
err := _main()
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
}
}
@@ -99,6 +99,17 @@ type RecordType struct {
Type string `json:"type"` // Type or record
}

// FsckOptions provides generic options on how to handle an fsck. Sane defaults
// will be used in lieu of options being provided.
type FsckOptions struct {
Verbose bool // Normal verbosity
PrintHashes bool // Prints every hash
Fix bool // Fix fixable errors

URL string // URL for dcrdata, used to verify anchors
File string // Path for results file
}

type Backend interface {
// Return timestamp information for given digests.
Get([][sha256.Size]byte) ([]GetResult, error)
@@ -123,4 +134,8 @@ type Backend interface {
// call may parint to stdout. The provided string describes the target
// location and is implementation specific.
Restore(*os.File, bool, string) error

// Fsck walks all data and verifies its integrity. In addition it
// verifies anchored timestamps' existence on the blockchain.
Fsck(*FsckOptions) error
}
@@ -58,8 +58,8 @@ func NewRestore(root string) (*FileSystem, error) {
return &FileSystem{root: root, db: db}, nil
}

func dumpDigestTimestamp(f *os.File, human bool, recordType string, dr backend.DigestReceived) error {
if human {
func dumpDigestTimestamp(f *os.File, verbose bool, recordType string, dr backend.DigestReceived) error {
if verbose {
ts := ts2dirname(dr.Timestamp)
fmt.Fprintf(f, "Digest : %v\n", dr.Digest)
fmt.Fprintf(f, "Timestamp : %v -> %v\n", dr.Timestamp, ts)
@@ -85,14 +85,14 @@ func dumpDigestTimestamp(f *os.File, human bool, recordType string, dr backend.D
return nil
}

func (fs *FileSystem) dumpGlobal(f *os.File, human bool) error {
func (fs *FileSystem) dumpGlobal(f *os.File, verbose bool) error {

i := fs.db.NewIterator(nil, nil)
defer i.Release()
for i.Next() {
key := hex.EncodeToString(i.Key())
value := int64(binary.LittleEndian.Uint64(i.Value()))
err := dumpDigestTimestamp(f, human,
err := dumpDigestTimestamp(f, verbose,
backend.RecordTypeDigestReceivedGlobal,
backend.DigestReceived{
Digest: key,
@@ -104,8 +104,20 @@ func (fs *FileSystem) dumpGlobal(f *os.File, human bool) error {
}
return i.Error()
}
func dumpFlushRecord(f *os.File, flushRecord *backend.FlushRecord) {
fmt.Fprintf(f, "Merkle root : %x\n",
flushRecord.Root)
fmt.Fprintf(f, "Tx : %v\n", flushRecord.Tx)
fmt.Fprintf(f, "Chain timestamp: %v\n",
flushRecord.ChainTimestamp)
fmt.Fprintf(f, "Flush timestamp: %v\n",
flushRecord.FlushTimestamp)
for _, v := range flushRecord.Hashes {
fmt.Fprintf(f, " Flushed : %x\n", *v)
}
}

func (fs *FileSystem) dumpTimestamp(f *os.File, human bool, ts int64) error {
func (fs *FileSystem) dumpTimestamp(f *os.File, verbose bool, ts int64) error {
db, err := fs.openRead(ts)
if err != nil {
return err
@@ -134,17 +146,8 @@ func (fs *FileSystem) dumpTimestamp(f *os.File, human bool, ts int64) error {
}

if flushRecord != nil {
if human {
fmt.Fprintf(f, "Merkle root : %x\n",
flushRecord.Root)
fmt.Fprintf(f, "Tx : %v\n", flushRecord.Tx)
fmt.Fprintf(f, "Chain timestamp: %v\n",
flushRecord.ChainTimestamp)
fmt.Fprintf(f, "Flush timestamp: %v\n",
flushRecord.FlushTimestamp)
for _, v := range flushRecord.Hashes {
fmt.Fprintf(f, " Hashes : %x\n", *v)
}
if verbose {
dumpFlushRecord(f, flushRecord)
} else {
e := json.NewEncoder(f)
rt := backend.RecordType{
@@ -171,7 +174,7 @@ func (fs *FileSystem) dumpTimestamp(f *os.File, human bool, ts int64) error {
}

for _, v := range digests {
err := dumpDigestTimestamp(f, human,
err := dumpDigestTimestamp(f, verbose,
backend.RecordTypeDigestReceived, v)
if err != nil {
return err
@@ -181,7 +184,7 @@ func (fs *FileSystem) dumpTimestamp(f *os.File, human bool, ts int64) error {
return nil
}

func (fs *FileSystem) dumpTimestamps(f *os.File, human bool) error {
func (fs *FileSystem) dumpTimestamps(f *os.File, verbose bool) error {
files, err := ioutil.ReadDir(fs.root)
if err != nil {
return err
@@ -201,11 +204,11 @@ func (fs *FileSystem) dumpTimestamps(f *os.File, human bool) error {
return fmt.Errorf("invalid timestamp: %v", fi.Name())
}

if human {
if verbose {
fmt.Fprintf(f, "--- Timestamp: %v %v\n", fi.Name(),
t.Unix())
}
err = fs.dumpTimestamp(f, human, t.Unix())
err = fs.dumpTimestamp(f, verbose, t.Unix())
if err != nil {
return err
}
@@ -214,15 +217,15 @@ func (fs *FileSystem) dumpTimestamps(f *os.File, human bool) error {
return nil
}

// Dump walks all directories and dumps the content to either human
// Dump walks all directories and dumps the content to either verbose
// readable or JSON format.
func (fs *FileSystem) Dump(f *os.File, human bool) error {
err := fs.dumpTimestamps(f, human)
func (fs *FileSystem) Dump(f *os.File, verbose bool) error {
err := fs.dumpTimestamps(f, verbose)
if err != nil {
return err
}
// Dump global
return fs.dumpGlobal(f, human)
return fs.dumpGlobal(f, verbose)
}

// restoreOpen opens/creates a leveldb based on the timestamp that is passed

0 comments on commit d496e2f

Please sign in to comment.
You can’t perform that action at this time.