Merge branch 'develop' into SOF-604

FactomProject · Mar 16, 2017 · 0f7236d · 0f7236d
2 parents 76eafca + 4302f5b
commit 0f7236d
Show file tree

Hide file tree

Showing 13 changed files with 828 additions and 72 deletions.
diff --git a/Utilities/DatabaseIntegrityCheck/DatabaseIntegrityCheck.go b/Utilities/DatabaseIntegrityCheck/DatabaseIntegrityCheck.go
@@ -170,6 +170,105 @@ func CheckDatabase(db interfaces.IDatabase) {
 	}
 
 	fmt.Printf("\tFinished looking for free-floating blocks\n")
+
+	fmt.Printf("\tLooking for missing EBlocks\n")
+
+	foundBlocks := 0
+	for _, dHash := range dBlocks {
+		dBlock, err := dbo.FetchDBlock(dHash)
+		if err != nil {
+			panic(err)
+		}
+		if dBlock == nil {
+			fmt.Printf("Could not find DBlock %v!", dHash.String())
+			panic("")
+		}
+		eBlockEntries := dBlock.GetEBlockDBEntries()
+		for _, v := range eBlockEntries {
+			eBlock, err := dbo.FetchEBlock(v.GetKeyMR())
+			if err != nil {
+				panic(err)
+			}
+			if eBlock == nil {
+				fmt.Errorf("Could not find eBlock %v!\n", v.GetKeyMR())
+			} else {
+				foundBlocks++
+			}
+		}
+	}
+
+	fmt.Printf("\tFinished looking for missing EBlocks - found %v\n", foundBlocks)
+
+	fmt.Printf("\tLooking for missing EBlock Entries\n")
+
+	chains, err := dbo.FetchAllEBlockChainIDs()
+	if err != nil {
+		panic(err)
+	}
+	checkCount := 0
+	missingCount := 0
+	for _, chain := range chains {
+		blocks, err := dbo.FetchAllEBlocksByChain(chain)
+		if err != nil {
+			panic(err)
+		}
+		if len(blocks) == 0 {
+			panic("Found no blocks!")
+		}
+		for _, block := range blocks {
+			entryHashes := block.GetEntryHashes()
+			if len(entryHashes) == 0 {
+				panic("Found no entryHashes!")
+			}
+			for _, eHash := range entryHashes {
+				if eHash.IsMinuteMarker() == true {
+					continue
+				}
+				entry, err := dbo.FetchEntry(eHash)
+				if err != nil {
+					panic(err)
+				}
+				if entry == nil {
+					missingCount++
+					fmt.Printf("Missing entry %v!\n", eHash.String())
+				} else {
+					checkCount++
+				}
+			}
+		}
+	}
+	fmt.Printf("Found %v entries, missing %v\n", checkCount, missingCount)
+	fmt.Printf("\tFinished looking for missing EBlock Entries\n")
+
+	//CheckMinuteNumbers(dbo)
+}
+
+func CheckMinuteNumbers(dbo interfaces.DBOverlay) {
+	fmt.Printf("\tChecking Minute Numbers\n")
+
+	ecBlocks, err := dbo.FetchAllECBlocks()
+	if err != nil {
+		panic(err)
+	}
+	for _, v := range ecBlocks {
+		entries := v.GetEntries()
+		found := 0
+		lastNumber := 0
+		for _, e := range entries {
+			if e.ECID() == entryCreditBlock.ECIDMinuteNumber {
+				number := int(e.(*entryCreditBlock.MinuteNumber).Number)
+				if number != lastNumber+1 {
+					fmt.Printf("Block #%v %v, Minute Number %v is not last minute plus 1\n", v.GetDatabaseHeight(), v.GetHash().String(), number)
+				}
+				lastNumber = number
+				found++
+			}
+		}
+		if found != 10 {
+			fmt.Printf("Block #%v %v only contains %v minute numbers\n", v.GetDatabaseHeight(), v.GetHash().String(), found)
+		}
+	}
+	fmt.Printf("\tFinished checking Minute Numbers\n")
 }
 
 type BlockSet struct {

diff --git a/Utilities/DatabasePorter/api.go b/Utilities/DatabasePorter/api.go
@@ -106,7 +106,7 @@ func GetEBlock(keymr string) (interfaces.IEntryBlock, error) {
 }
 
 func GetEntry(hash string) (interfaces.IEBEntry, error) {
-	for i := 0; i < 100; i++ {
+	for i := 0; i < 10; i++ {
 		raw, err := GetRaw(hash)
 		if err != nil {
 			fmt.Printf("got error %s\n", err)
@@ -130,7 +130,7 @@ func GetEntry(hash string) (interfaces.IEBEntry, error) {
 		}
 		return entry, nil
 	}
-	panic("Failed 100 times to get the data " + hash)
+	//panic("Failed 100 times to get the data " + hash)
 	return nil, nil
 }
 

diff --git a/Utilities/DatabasePorter/porter.go b/Utilities/DatabasePorter/porter.go
@@ -267,10 +267,11 @@ func SaveBlocksLoop(input chan []interfaces.IDirectoryBlock, done chan int) {
 									fmt.Printf("Problem getting entry `%v` from block %v\n", ehash.String(), e.GetKeyMR().String())
 									panic(err)
 								}
-
-								blockSet.Mutex.Lock()
-								blockSet.Entries = append(blockSet.Entries, entry)
-								blockSet.Mutex.Unlock()
+								if entry != nil {
+									blockSet.Mutex.Lock()
+									blockSet.Entries = append(blockSet.Entries, entry)
+									blockSet.Mutex.Unlock()
+								}
 							}(eHash)
 						}
 						for range eBlockEntries {

diff --git a/common/directoryBlock/directoryBlock.go b/common/directoryBlock/directoryBlock.go
@@ -41,8 +41,8 @@ func (c *DirectoryBlock) Init() {
 }
 
 func (c *DirectoryBlock) SetEntryHash(hash, chainID interfaces.IHash, index int) {
-	if len(c.DBEntries) < index {
-		ent := make([]interfaces.IDBEntry, index)
+	if len(c.DBEntries) <= index {
+		ent := make([]interfaces.IDBEntry, index+1)
 		copy(ent, c.DBEntries)
 		c.DBEntries = ent
 	}

diff --git a/common/directoryBlock/directoryBlockEntry_test.go b/common/directoryBlock/directoryBlockEntry_test.go
@@ -5,10 +5,10 @@
 package directoryBlock_test
 
 import (
-	"testing"
-
+	"fmt"
 	. "github.com/FactomProject/factomd/common/directoryBlock"
 	"github.com/FactomProject/factomd/common/primitives"
+	"testing"
 )
 
 func TestUnmarshalNilDBEntry(t *testing.T) {
@@ -114,3 +114,72 @@ func TestDBSEMarshalUnmarshal(t *testing.T) {
 		}
 	}
 }
+
+func TestHash(t *testing.T) {
+	dbe := new(DBEntry)
+
+	h, _ := primitives.HexToHash("3e3eb61fb20e71d8211882075d404f5929618a189d23aba8c892b22228aa0d71")
+	dbe.SetChainID(h)
+	h, _ = primitives.HexToHash("9daad42e5efedf3075fa2cf51908babdb568f431a3c13b9a496ffbfb7160ad2e")
+	dbe.SetKeyMR(h)
+	hash := dbe.ShaHash()
+
+	keymr, _ := primitives.HexToHash("7509a84bcda2045a400b4650135613685449e05b6b1cb578f152ae4682d9d6ea")
+
+	if !hash.IsSameAs(keymr) {
+		fmt.Println(hash)
+		fmt.Println(keymr)
+		t.Fail()
+	}
+}
+
+func TestPrintsE(t *testing.T) {
+	dbe := new(DBEntry)
+	h, _ := primitives.HexToHash("3e3eb61fb20e71d8211882075d404f5929618a189d23aba8c892b22228aa0d71")
+	dbe.SetChainID(h)
+	h, _ = primitives.HexToHash("9daad42e5efedf3075fa2cf51908babdb568f431a3c13b9a496ffbfb7160ad2e")
+	dbe.SetKeyMR(h)
+	returnVal := dbe.String()
+
+	expectedString := `ChainID: 3e3eb61fb20e71d8211882075d404f5929618a189d23aba8c892b22228aa0d71
+      KeyMR:   9daad42e5efedf3075fa2cf51908babdb568f431a3c13b9a496ffbfb7160ad2e
+`
+
+	if returnVal != expectedString {
+		fmt.Println(returnVal)
+		fmt.Println(expectedString)
+		t.Fail()
+	}
+
+	returnVal, _ = dbe.JSONString()
+	//fmt.Println(returnVal)
+
+	expectedString = `{"ChainID":"3e3eb61fb20e71d8211882075d404f5929618a189d23aba8c892b22228aa0d71","KeyMR":"9daad42e5efedf3075fa2cf51908babdb568f431a3c13b9a496ffbfb7160ad2e"}`
+	if returnVal != expectedString {
+		fmt.Println("got", returnVal)
+		fmt.Println("expected", expectedString)
+		t.Fail()
+	}
+
+	returnBytes, _ := dbe.JSONByte()
+	s := string(returnBytes)
+	if s != expectedString {
+		fmt.Println("got", s)
+		fmt.Println("expected", expectedString)
+		t.Fail()
+	}
+}
+
+func TestCheckErrorsMarshal(t *testing.T) {
+	dbe := new(DBEntry)
+
+	h, _ := primitives.HexToHash("3e3eb61fb20e71d8211882075d404f5929618a189d23aba8c892b22228aa0d71")
+	dbe.SetChainID(h)
+
+	_, err := dbe.MarshalBinary()
+	if err != nil {
+		fmt.Println("expected better revocery from missing keymr", err)
+		t.Fail()
+	}
+
+}