db.go

// Copyright 2017 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"archive/tar"
	"bytes"
	"encoding/binary"
	"encoding/hex"
	"fmt"
	"io"
	"os"
	"path/filepath"

	"github.com/ethereum/go-ethereum/cmd/utils"
	"github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/log"
	"github.com/ethereum/go-ethereum/rlp"
	"github.com/ethersphere/swarm/chunk"
	"github.com/ethersphere/swarm/storage/localstore"
	"github.com/syndtr/goleveldb/leveldb"
	"github.com/syndtr/goleveldb/leveldb/opt"
	"gopkg.in/urfave/cli.v1"
)

var legacyKeyIndex = byte(0)
var keyData = byte(6)

type dpaDBIndex struct {
	Idx    uint64
	Access uint64
}

var dbCommand = cli.Command{
	Name:               "db",
	CustomHelpTemplate: helpTemplate,
	Usage:              "manage the local chunk database",
	ArgsUsage:          "db COMMAND",
	Description:        "Manage the local chunk database",
	Subcommands: []cli.Command{
		{
			Action:             dbExport,
			CustomHelpTemplate: helpTemplate,
			Name:               "export",
			Usage:              "export a local chunk database as a tar archive (use - to send to stdout)",
			ArgsUsage:          "<chunkdb> <file>",
			Description: `
Export a local chunk database as a tar archive (use - to send to stdout).

    swarm db export ~/.ethereum/swarm/bzz-KEY/chunks chunks.tar

The export may be quite large, consider piping the output through the Unix
pv(1) tool to get a progress bar:

    swarm db export ~/.ethereum/swarm/bzz-KEY/chunks - | pv > chunks.tar
`,
		},
		{
			Action:             dbImport,
			CustomHelpTemplate: helpTemplate,
			Name:               "import",
			Usage:              "import chunks from a tar archive into a local chunk database (use - to read from stdin)",
			ArgsUsage:          "<chunkdb> <file>",
			Description: `Import chunks from a tar archive into a local chunk database (use - to read from stdin).

    swarm db import ~/.ethereum/swarm/bzz-KEY/chunks chunks.tar

The import may be quite large, consider piping the input through the Unix
pv(1) tool to get a progress bar:

    pv chunks.tar | swarm db import ~/.ethereum/swarm/bzz-KEY/chunks -`,
			Flags: []cli.Flag{
				SwarmLegacyFlag,
			},
		},
	},
}

func dbExport(ctx *cli.Context) {
	args := ctx.Args()
	if len(args) != 3 {
		utils.Fatalf("invalid arguments, please specify both <chunkdb> (path to a local chunk database), <file> (path to write the tar archive to, - for stdout) and the base key")
	}

	var out io.Writer
	if args[1] == "-" {
		out = os.Stdout
	} else {
		f, err := os.Create(args[1])
		if err != nil {
			utils.Fatalf("error opening output file: %s", err)
		}
		defer f.Close()
		out = f
	}

	isLegacy := localstore.IsLegacyDatabase(args[0])
	if isLegacy {
		count, err := exportLegacy(args[0], common.Hex2Bytes(args[2]), out)
		if err != nil {
			utils.Fatalf("error exporting legacy local chunk database: %s", err)
		}

		log.Info(fmt.Sprintf("successfully exported %d chunks from legacy db", count))
		return
	}

	store, err := openLDBStore(args[0], common.Hex2Bytes(args[2]))
	if err != nil {
		utils.Fatalf("error opening local chunk database: %s", err)
	}
	defer store.Close()

	count, err := store.Export(out)
	if err != nil {
		utils.Fatalf("error exporting local chunk database: %s", err)
	}

	log.Info(fmt.Sprintf("successfully exported %d chunks", count))
}

func dbImport(ctx *cli.Context) {
	args := ctx.Args()
	if len(args) != 3 {
		utils.Fatalf("invalid arguments, please specify both <chunkdb> (path to a local chunk database), <file> (path to read the tar archive from, - for stdin) and the base key")
	}

	legacy := ctx.IsSet(SwarmLegacyFlag.Name)

	store, err := openLDBStore(args[0], common.Hex2Bytes(args[2]))
	if err != nil {
		utils.Fatalf("error opening local chunk database: %s", err)
	}
	defer store.Close()

	var in io.Reader
	if args[1] == "-" {
		in = os.Stdin
	} else {
		f, err := os.Open(args[1])
		if err != nil {
			utils.Fatalf("error opening input file: %s", err)
		}
		defer f.Close()
		in = f
	}

	count, err := store.Import(in, legacy)
	if err != nil {
		utils.Fatalf("error importing local chunk database: %s", err)
	}

	log.Info(fmt.Sprintf("successfully imported %d chunks", count))
}

func openLDBStore(path string, basekey []byte) (*localstore.DB, error) {
	if _, err := os.Stat(filepath.Join(path, "CURRENT")); err != nil {
		return nil, fmt.Errorf("invalid chunkdb path: %s", err)
	}

	return localstore.New(path, basekey, nil)
}

func decodeIndex(data []byte, index *dpaDBIndex) error {
	dec := rlp.NewStream(bytes.NewReader(data), 0)
	return dec.Decode(index)
}

func getDataKey(idx uint64, po uint8) []byte {
	key := make([]byte, 10)
	key[0] = keyData
	key[1] = po
	binary.BigEndian.PutUint64(key[2:], idx)

	return key
}

func exportLegacy(path string, basekey []byte, out io.Writer) (int64, error) {
	tw := tar.NewWriter(out)
	defer tw.Close()
	db, err := leveldb.OpenFile(path, &opt.Options{OpenFilesCacheCapacity: 128})
	if err != nil {
		return 0, err
	}
	defer db.Close()

	it := db.NewIterator(nil, nil)
	defer it.Release()
	var count int64
	for ok := it.Seek([]byte{legacyKeyIndex}); ok; ok = it.Next() {
		key := it.Key()
		if (key == nil) || (key[0] != legacyKeyIndex) {
			break
		}

		var index dpaDBIndex

		hash := key[1:]
		decodeIndex(it.Value(), &index)

		po := uint8(chunk.Proximity(basekey, hash))

		datakey := getDataKey(index.Idx, po)
		data, err := db.Get(datakey, nil)
		if err != nil {
			log.Warn(fmt.Sprintf("Chunk %x found but could not be accessed: %v, %x", key, err, datakey))
			continue
		}

		hdr := &tar.Header{
			Name: hex.EncodeToString(hash),
			Mode: 0644,
			Size: int64(len(data)),
		}
		if err := tw.WriteHeader(hdr); err != nil {
			return count, err
		}
		if _, err := tw.Write(data); err != nil {
			return count, err
		}
		count++
	}

	return count, nil
}