merge with FD-1111_xuan_rollup-1

README.md

@@ -66,89 +66,186 @@ To get the current list of flags, type the command:
 
 Which will get you something like:
 
-    //////////////////////// Copyright 2017 Factom Foundation
-    //////////////////////// Use of this source code is governed by the MIT
-    //////////////////////// license that can be found in the LICENSE file.
-    Go compiler version: go1.6.2
-    Using build: 
-    len(Args) 2
+    Command Line Arguments:
+        -h
     Usage of factomd:
+    -balancehash
+        If false, then don't pass around balance hashes (default true)
     -blktime int
-            Seconds per block.  Production is 600.
+        Seconds per block.  Production is 600.
+    -broadcastnum int
+        Number of peers to broadcast to in the peer to peer networking (default 16)
+    -checkheads
+        Enables checking chain heads on boot (default true)
     -clonedb string
-            Override the main node and use this database for the clones in a Network.
+        Override the main node and use this database for the clones in a Network.
+    -config string
+        Override the config file location (factomd.conf)
+    -controlpanelport int
+        Port for control panel webserver;  Default 8090
+    -controlpanelsetting string
+        Can set to 'disabled', 'readonly', or 'readwrite' to overwrite config file
     -count int
-            The number of nodes to generate (default 1)
+        The number of nodes to generate (default 1)
+    -customnet string
+        This string specifies a custom blockchain network ID.
     -db string
-            Override the Database in the Config file and use this Database implementation
+        Override the Database in the Config file and use this Database implementation. Options Map, LDB, or Bolt
+    -deadline int
+        Timeout Delay in milliseconds used on Reads and Writes to the network comm (default 1000)
+    -debugconsole string
+        Enable DebugConsole on port. localhost:8093 open 8093 and spawns a telnet console, remotehost:8093 open 8093
+    -debuglog string
+        regex to pick which logs to save
     -drop int
-            Number of messages to drop out of every thousand
+        Number of messages to drop out of every thousand
+    -enablenet
+        Enable or disable networking (default true)
     -exclusive
-            If true, we only dial out to special/trusted peers.
-    -folder string
-            Directory in .factom to store nodes. (eg: multiple nodes on one filesystem support)
+        If true, we only dial out to special/trusted peers.
+    -exclusive_in
+        If true, we only dial out to special/trusted peers and no incoming connections are accepted.
+    -exposeprofiler
+        Setting this exposes the profiling port to outside localhost.
+    -factomhome string
+        Set the Factom home directory. The .factom folder will be placed here if set, otherwise it will default to $HOME
+    -fast
+        If true, Factomd will fast-boot from a file. (default true)
+    -fastlocation string
+        Directory to put the Fast-boot file in.
+    -fastsaverate int
+        Save a fastboot file every so many blocks. Should be > 1000 for live systems. (default 1000)
+    -faulttimeout int
+        Seconds before considering Federated servers at-fault. Default is 120. (default 120)
+    -fixheads
+        If --checkheads is enabled, then this will also correct any errors reported (default true)
+    -fnet string
+        Read the given file to build the network connections
     -follower
-            If true, force node to be a follower.  Only used when replaying a journal.
+        If true, force node to be a follower.  Only used when replaying a journal.
+    -fullhasheslog
+        true create a log of all unique hashes seen during processing
     -journal string
-            Rerun a Journal of messages
+        Rerun a Journal of messages
+    -journaling
+        Write a journal of all messages received. Default is off.
+    -keepmismatch
+        If true, do not discard DBStates even when a majority of DBSignatures have a different hash
     -leader
-            If true, force node to be a leader.  Only used when replaying a journal. (default true)
+        If true, force node to be a leader.  Only used when replaying a journal. (default true)
+    -logPort string
+        Port for pprof logging (default "6060")
+    -logjson
+        Use to set logging to use a json formatting
+    -loglvl string
+        Set log level to either: none, debug, info, warning, error, fatal or panic (default "none")
+    -logstash
+        If true, use Logstash
+    -logurl string
+        Endpoint URL for Logstash (default "localhost:8345")
+    -mpr int
+        Set the Memory Profile Rate to update profiling per X bytes allocated. Default 512K, set to 1 to profile everything, 0 to disable. (default 524288)
     -net string
-            The default algorithm to build the network connections (default "tree")
+        The default algorithm to build the network connections (default "tree")
+    -network string
+        Network to join: MAIN, TEST or LOCAL
+    -networkport int
+        Port for p2p network; default 8110
     -node int
-            Node Number the simulator will set as the focus
-    -p2pPort string
-            Port to listen for peers on. (default "8108")
+        Node Number the simulator will set as the focus
+    -nodename string
+        Assign a name to the node
     -peers string
-            Array of peer addresses. 
+        Array of peer addresses. 
+    -plugin string
+        Input the path to any plugin binaries
     -port int
-            Address to serve WSAPI on
+        Port where we serve WSAPI;  default 8088
     -prefix string
-            Prefix the Factom Node Names with this value; used to create leaderless networks.
-    -profile string
-            If true, turn on the go Profiler to profile execution of Factomd
+        Prefix the Factom Node Names with this value; used to create leaderless networks.
+    -reparseanchorchains
+        If true, reparse bitcoin and ethereum anchor chains in the database
     -rotate
-            If true, responsiblity is owned by one leader, and rotated over the leaders.
+        If true, responsibility is owned by one leader, and Rotated over the leaders.
+    -roundtimeout int
+        Seconds before audit servers will increment rounds and volunteer. (default 30)
+    -rpcpass string
+        Password to protect factomd local API. Ignored if rpcuser is blank
+    -rpcuser string
+        Username to protect factomd local API with simple HTTP authentication
     -runtimeLog
-            If true, maintain runtime logs of messages passed.
-    -test.bench string
-            regular expression to select benchmarks to run
+        If true, maintain runtime logs of messages passed.
+    -selfaddr string
+        comma separated IPAddresses and DNS names of this factomd to use when creating a cert file
+    -sim_stdin
+        If true, sim control reads from stdin. (default true)
+    -startdelay int
+        Delay to start processing messages, in seconds (default 10)
+    -stderrlog string
+        Log stderr to a file, optionally the same file as stdout
+    -stdoutlog string
+        Log stdout to a file
+    -sync2 int
+        Set the initial blockheight for the second Sync pass. Used to force a total sync, or skip unnecessary syncing of entries. (default -1)
+    -test.bench regexp
+        run only benchmarks matching regexp
     -test.benchmem
-            print memory allocations for benchmarks
-    -test.benchtime duration
-            approximate run time for each benchmark (default 1s)
-    -test.blockprofile string
-            write a goroutine blocking profile to the named file after execution
-    -test.blockprofilerate int
-            if >= 0, calls runtime.SetBlockProfileRate() (default 1)
+        print memory allocations for benchmarks
+    -test.benchtime d
+        run each benchmark for duration d (default 1s)
+    -test.blockprofile file
+        write a goroutine blocking profile to file
+    -test.blockprofilerate rate
+        set blocking profile rate (see runtime.SetBlockProfileRate) (default 1)
     -test.count n
-            run tests and benchmarks n times (default 1)
-    -test.coverprofile string
-            write a coverage profile to the named file after execution
-    -test.cpu string
-            comma-separated list of number of CPUs to use for each test
-    -test.cpuprofile string
-            write a cpu profile to the named file during execution
-    -test.memprofile string
-            write a memory profile to the named file after execution
-    -test.memprofilerate int
-            if >=0, sets runtime.MemProfileRate
-    -test.outputdir string
-            directory in which to write profiles
-    -test.parallel int
-            maximum test parallelism (default 1)
-    -test.run string
-            regular expression to select tests and examples to run
+        run tests and benchmarks n times (default 1)
+    -test.coverprofile file
+        write a coverage profile to file
+    -test.cpu list
+        comma-separated list of cpu counts to run each test with
+    -test.cpuprofile file
+        write a cpu profile to file
+    -test.failfast
+        do not start new tests after the first test failure
+    -test.list regexp
+        list tests, examples, and benchmarks matching regexp then exit
+    -test.memprofile file
+        write an allocation profile to file
+    -test.memprofilerate rate
+        set memory allocation profiling rate (see runtime.MemProfileRate)
+    -test.mutexprofile string
+        write a mutex contention profile to the named file after execution
+    -test.mutexprofilefraction int
+        if >= 0, calls runtime.SetMutexProfileFraction() (default 1)
+    -test.outputdir dir
+        write profiles to dir
+    -test.parallel n
+        run at most n tests in parallel (default 8)
+    -test.run regexp
+        run only tests and examples matching regexp
     -test.short
-            run smaller test suite to save time
-    -test.timeout duration
-            if positive, sets an aggregate time limit for all tests
-    -test.trace string
-            write an execution trace to the named file after execution
+        run smaller test suite to save time
+    -test.testlogfile file
+        write test action log to file (for use only by cmd/go)
+    -test.timeout d
+        panic test binary after duration d (default 0, timeout disabled)
+    -test.trace file
+        write an execution trace to file
     -test.v
-            verbose: print additional output
+        verbose: print additional output
     -timedelta int
-            Maximum timeDelta in milliseconds to offset each node.  Simulates deltas in system clocks over a network.
+        Maximum timeDelta in milliseconds to offset each node.  Simulates deltas in system clocks over a network.
+    -tls
+        Set to true to require encrypted connections to factomd API and Control Panel
+    -tormanage
+        Use torrent dbstate manager. Must have plugin binary installed and in $PATH
+    -torupload
+        Be a torrent uploader
+    -waitentries
+        Wait for Entries to be validated prior to execution of messages
+    -wrproc
+        Write processed blocks to temporary debug file (default true)
+
 
 The flags that begin with "test." are supplied by the profiling package installed.  The flags that relate to running factomd and the simulator are the following, with a little more explaination.  That follows below.
 

Utilities/DatabaseGenerator/blockgen/generator.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/FactomProject/factomd/common/constants"
+	"github.com/FactomProject/factomd/common/identity"
 	"github.com/FactomProject/factomd/common/interfaces"
 	"github.com/FactomProject/factomd/common/messages"
 	"github.com/FactomProject/factomd/common/messages/electionMsgs"
@@ -71,7 +72,9 @@ func NewDBGenerator(c *DBGeneratorConfig) (*DBGenerator, error) {
 
 func NewGeneratorState(conf *DBGeneratorConfig, starttime interfaces.Timestamp) *state.State {
 	s := new(state.State)
+	s.TimestampAtBoot = starttime
 	s.SetLeaderTimestamp(starttime)
+	s.Balancehash = primitives.NewZeroHash()
 	var db interfaces.IDatabase
 	var err error
 	switch strings.ToLower(conf.DBType) {
@@ -137,12 +140,15 @@ func (g *DBGenerator) loadGenesis() {
 	g.FactomdState.DBStates.SaveDBStateToDB(sds)
 	sds.Saved = true
 	g.last = sds
+	g.FactomdState.DBStates.Last().Saved = true
 }
 
 // SaveDBState will save a dbstate to disk
 func (g *DBGenerator) SaveDBState(dbstate *state.DBState) {
 	dbstate.ReadyToSave = true
 	dbstate.Signed = true
+	dbstate.SaveStruct = new(state.SaveState)
+	dbstate.SaveStruct.IdentityControl = identity.NewIdentityManager()
 	g.FactomdState.DBStates.ProcessHeight = dbstate.DirectoryBlock.GetDatabaseHeight()
 	put := g.FactomdState.DBStates.Put(dbstate)
 	if !put {
@@ -152,6 +158,17 @@ func (g *DBGenerator) SaveDBState(dbstate *state.DBState) {
 	if !progress {
 		log.Warnf("%d Not saved to disk", dbstate.DirectoryBlock.GetDatabaseHeight())
 	}
+
+EntryLoop:
+	for {
+		select {
+		case ent := <-g.FactomdState.WriteEntry:
+			g.FactomdState.GetDB().InsertEntry(ent)
+		default:
+			break EntryLoop
+		}
+	}
+
 	dbstate.Saved = true
 	g.FactomdState.DBStates.Complete = dbstate.DirectoryBlock.GetDatabaseHeight() - g.FactomdState.DBStates.Base
 	g.FactomdState.ProcessLists.DBHeightBase = dbstate.DirectoryBlock.GetDatabaseHeight()
@@ -236,6 +253,7 @@ func (g *DBGenerator) CreateBlocks(amt int) error {
 
 		loopEntries += len(dbstate.Entries)
 		totalEntries += len(dbstate.Entries)
+
 		g.SaveDBState(dbstate)
 		g.last = dbstate
 

Utilities/DatabaseGenerator/main.go

@@ -22,7 +22,7 @@ func main() {
 		loglvl     = flag.String("loglvl", "info", "Sets log level to 'debug', 'info', 'warning', or 'error'")
 		configfile = flag.String("config", "", "Generator config file location.")
 		genconfig  = flag.Bool("genconfig", false, "Does not run the program, but instead outputs the default config file")
-		profiling  = flag.Bool("profile", false, "Turn on profiling on :6060")
+		profiling  = flag.Bool("profile", true, "Turn on profiling on :6060")
 	)
 	flag.IntVar(&blockcount, "b", 1000, "Number of blocks to generate")
 

Utilities/DatabaseGenerator/smallblocks.yaml

@@ -0,0 +1,21 @@
+dbpath: factoid_level.db
+dbtype: level
+factomdconfigpath: gen.conf
+customnetid: gen
+starttime: ""
+loopsperprint: 200
+entrygenerator: "incr"
+entrygenconfig:
+  entriespereblock:
+    min: 5
+    max: 10
+  entrysize:
+    min: 100
+    max: 250
+  eblocksperheight:
+    min: 1
+    max: 5
+  multithreaded: true
+  threadpoolcount: 8
+
+

Utilities/DatabasePorter/porter.go

@@ -191,7 +191,7 @@ func main() {
 	}
 
 	fmt.Printf("\t\tRebulding DirBlockInfo\n")
-	err = dbo.RebuildDirBlockInfo()
+	err = dbo.ReparseAnchorChains()
 	if err != nil {
 		panic(err)
 	}

anchor/anchorRecord.go

@@ -20,9 +20,14 @@ import (
 //AnchorRecord is used to construct anchor chain
 type AnchorRecord struct {
 	AnchorRecordVer int
-	DBHeight        uint32
-	KeyMR           string
-	RecordHeight    uint32 //the block height we intended to put the anchorrecod into
+	DBHeight        uint32 `json:",omitempty"` // The only directory block height included in this anchor
+	KeyMR           string `json:",omitempty"` // Merkle root of the only directory block included in this anchor
+
+	DBHeightMax uint32 `json:",omitempty"` // The highest directory block height included in this anchor window
+	DBHeightMin uint32 `json:",omitempty"` // The lowest directory block height included in this anchor window
+	WindowMR    string `json:",omitempty"` // Merkle root of all directory block KeyMRs from DBHeightMin to DBHeightMax
+
+	RecordHeight uint32 // Directory block height we intended to put the AnchorRecord into
 
 	Bitcoin  *BitcoinStruct  `json:",omitempty"`
 	Ethereum *EthereumStruct `json:",omitempty"`
@@ -37,11 +42,11 @@ type BitcoinStruct struct {
 }
 
 type EthereumStruct struct {
-	Address     string //0x30aa981f6d2fce81083e584c8ee2f822b548752f
-	TXID        string //0x50ea0effc383542811a58704a6d6842ed6d76439a2d942d941896ad097c06a78
-	BlockHeight int64  //293003
-	BlockHash   string //0x3b504616495fc9cf7be9b5b776692a9abbfb95491fa62abf62dcdf4d53ff5979
-	Offset      int64  //0
+	ContractAddress string // Address of the Ethereum anchor contract
+	TxID            string // Transaction ID of this particular anchor
+	BlockHeight     int64  // Ethereum block height that this anchor was included in
+	BlockHash       string // Hash of the Ethereum block that this anchor was included in
+	TxIndex         int64  // Where the anchor tx is located within that block
 }
 
 var _ interfaces.Printable = (*AnchorRecord)(nil)

common/constants/ack.go

@@ -4,6 +4,14 @@
 
 package constants
 
+const (
+	// MaxAckHeightMinuteDelta is the maximum number of minute in the
+	//	future we will set our HighestAckHeight too. This means
+	// 	2000/10 = max number of blocks to set the max height too ontop
+	//	of our current block height.
+	MaxAckHeightMinuteDelta = 2000
+)
+
 // Ack status levels
 const (
 	_ int = iota