74 gpu rank calculation

.circleci/config.yml

@@ -15,7 +15,7 @@ aliases:
   - &master_filter
     filters:
       branches:
-        only: /.*/
+        only: master
 
 jobs:
 
@@ -50,9 +50,18 @@ jobs:
           command: |
             cd cosmos/poc
             docker build -t build/cyberd ./
-            docker login -u $DOCKER_USER -p $DOCKER_PASS
-            docker tag build/cyberd cybernode/cyberd:$CIRCLE_BRANCH
-            docker push cybernode/cyberd:$CIRCLE_BRANCH
+            if [[  -n  "$CIRCLE_TAG" ]]; then
+              docker login -u $DOCKER_USER -p $DOCKER_PASS
+              docker tag build/cyberd cybernode/cyberd:latest
+              docker push cybernode/cyberd:latest
+              docker tag build/cyberd cybernode/cyberd:$CIRCLE_TAG
+              docker push cybernode/cyberd:$CIRCLE_TAG
+            else
+              docker tag build/cyberd cybernode/cyberd:$CIRCLE_BRANCH
+              docker push cybernode/cyberd:$CIRCLE_BRANCH
+            fi
+
+
 
   update_changelog:
     <<: *defaults
@@ -140,7 +149,11 @@ workflows:
           requires:
             - build_daemon_and_cli
       - build_and_deploy_images_to_dockerhub:
-          <<: *master_filter
+          filters:
+            tags:
+              only: /.*/
+            branches:
+              only: master
           requires:
             - build_daemon_and_cli
       - release_cyberd_and_cyberdcli_binaries:

.gitignore

@@ -12,3 +12,5 @@ node_modules/
 initial-rank
 initial_guess_impact_results
 *enwiki*
+*.so
+*.o

cosmos/poc/app/app.go

@@ -15,6 +15,7 @@ import (
 	cmn "github.com/tendermint/tendermint/libs/common"
 	dbm "github.com/tendermint/tendermint/libs/db"
 	"github.com/tendermint/tendermint/libs/log"
+	"math"
 	"os"
 	"time"
 )
@@ -61,6 +62,8 @@ type CyberdApp struct {
 	memStorage      *InMemoryStorage
 
 	latestRankHash []byte
+
+	computeUnit rank.ComputeUnit
 }
 
 // NewBasecoinApp returns a reference to a new CyberdApp given a
@@ -69,7 +72,9 @@ type CyberdApp struct {
 // In addition, all necessary mappers and keepers are created, routes
 // registered, and finally the stores being mounted along with any necessary
 // chain initialization.
-func NewCyberdApp(logger log.Logger, db dbm.DB, baseAppOptions ...func(*baseapp.BaseApp)) *CyberdApp {
+func NewCyberdApp(
+	logger log.Logger, db dbm.DB, computeUnit rank.ComputeUnit, baseAppOptions ...func(*baseapp.BaseApp),
+) *CyberdApp {
 	// create and register app-level codec for TXs and accounts
 	cdc := MakeCodec()
 
@@ -95,6 +100,7 @@ func NewCyberdApp(logger log.Logger, db dbm.DB, baseAppOptions ...func(*baseapp.
 		dbKeys:          dbKeys,
 		persistStorages: storages,
 		mainStorage:     ms,
+		computeUnit:     computeUnit,
 	}
 
 	// define and attach the mappers and keepers
@@ -142,12 +148,12 @@ func (app *CyberdApp) EndBlocker(ctx sdk.Context, _ abci.RequestEndBlock) abci.R
 
 	start := time.Now()
 	app.BaseApp.Logger.Info("Calculating rank")
-	newRank, steps := rank.CalculateRank(app.memStorage)
+	newRank, steps := rank.CalculateRank(app.memStorage, app.computeUnit)
 	app.BaseApp.Logger.Info("Rank calculated", "steps", steps, "time", time.Since(start))
 
 	rankAsBytes := make([]byte, 8*len(newRank))
-	for i, ui64 := range newRank {
-		binary.LittleEndian.PutUint64(rankAsBytes[i*8:i*8+8], ui64)
+	for i, f64 := range newRank {
+		binary.LittleEndian.PutUint64(rankAsBytes[i*8:i*8+8], math.Float64bits(f64))
 	}
 
 	hash := sha256.Sum256(rankAsBytes)

cosmos/poc/app/rank/README.md

@@ -0,0 +1,46 @@
+# Cuda support
+
+## Running with GPU rank calculation
+To compile cyberd with cuda support first install latest [cuda toolkit](https://developer.nvidia.com/cuda-downloads).
+
+Next compile **cbdrank** lib, copy it to `/usr/lib/` folder:
+
+```bash
+# project root
+cd poc/app/rank/cuda
+nvcc -fmad=false -shared -o libcbdrank.so rank.cu --compiler-options '-fPIC -frounding-math -fsignaling-nans'
+sudo cp libcbdrank.so /usr/lib/
+sudo cp cbdrank.h /usr/lib/
+```
+
+Compile binaries, copy configs and run daemon
+```bash
+# project root
+cd poc
+cp testnet/genesis.json .cyberd/config/genesis.json
+cp testnet/config.toml .cyberd/config/config.toml
+go build -tags cuda -o daemon ./cyberd
+./cyberd
+```
+
+## Testing 
+To test GPU and CPU rank computing determinism run:
+```bash
+# project root
+cd poc/app/rank/cuda
+nvcc -fmad=false -shared -o libcbdrank.so rank.cu --compiler-options '-fPIC -frounding-math -fsignaling-nans'
+sudo cp libcbdrank.so /usr/lib/
+sudo cp cbdrank.h /usr/lib/
+go build -tags cuda -o test *.go && ./test && rm test
+```
+After executing check ranks. They should match.
+
+## Compilation args
+
+While creating the shared libraries, position independent code should be produced. This helps the shared library 
+ to get loaded as any address instead of some fixed address. For this `-fPIC` option is used.
+
+See https://gcc.gnu.org/wiki/FloatingPointMath. '-frounding-math' is round-to-zero for all floating point to integer
+ conversions, and round-to-nearest for all other arithmetic truncations. FMA is disabled for current version. 
+
+

cosmos/poc/app/rank/calculate.go

@@ -1,107 +1,18 @@
 package rank
 
-import (
-	. "github.com/cybercongress/cyberd/cosmos/poc/app/storage"
-	"sync"
-)
+import . "github.com/cybercongress/cyberd/cosmos/poc/app/storage"
+
+type ComputeUnit int
 
 const (
-	one       uint64 = 1000000000 // represents 1.000000000
-	d         uint64 = 850000000  // represents 0.850000000
-	tolerance uint64 = 10000000   // represents 0.010000000
+	CPU ComputeUnit = iota
+	GPU ComputeUnit = iota
 )
 
-func CalculateRank(data *InMemoryStorage) ([]uint64, int) {
-
-	size := data.GetCidsCount()
-
-	if size == 0 {
-		return []uint64{}, 0
-	}
-
-	prevrank := make([]uint64, size)
-
-	tOverSize := (one - d) / size
-	danglingNodes := calculateDanglingNodes(data)
-
-	for _, i := range danglingNodes {
-		prevrank[i] = tOverSize
-	}
-
-	change := 2 * one
-
-	steps := 0
-	var rank []uint64
-	for change > tolerance {
-		rank = step(tOverSize, prevrank, danglingNodes, data)
-		change = calculateChange(prevrank, rank)
-		prevrank = rank
-		steps++
-	}
-
-	return rank, steps
-}
-
-func calculateDanglingNodes(data *InMemoryStorage) []int64 {
-
-	cidsCount := data.GetCidsCount()
-	outLinks := data.GetInLinks()
-	danglingNodes := make([]int64, 0)
-
-	i := uint64(0)
-	for i < cidsCount {
-		if len(outLinks[CidNumber(i)]) == 0 {
-			danglingNodes = append(danglingNodes, int64(i))
-		}
-		i++
+func CalculateRank(data *InMemoryStorage, unit ComputeUnit) ([]float64, int) {
+	if unit == CPU {
+		return calculateRankCPU(data)
+	} else {
+		return calculateRankGPU(data)
 	}
-
-	return danglingNodes
-}
-
-func step(tOverSize uint64, prevrank []uint64, danglingNodes []int64, data *InMemoryStorage) []uint64 {
-
-	innerProduct := uint64(0)
-	for _, danglingNode := range danglingNodes {
-		innerProduct += prevrank[danglingNode]
-	}
-
-	innerProductOverSize := innerProduct / uint64(len(prevrank))
-	rank := append(make([]uint64, 0, len(prevrank)), prevrank...)
-
-	var wg sync.WaitGroup
-	wg.Add(len(data.GetInLinks()))
-
-	for i, inLinksForI := range data.GetInLinks() {
-
-		go func(cid CidNumber, inLinks CidLinks) {
-			defer wg.Done()
-			ksum := uint64(0)
-
-			for j := range inLinks {
-				linkStake := data.GetOverallLinkStake(CidNumber(j), CidNumber(cid))
-				jCidOutStake := data.GetOverallOutLinksStake(CidNumber(j))
-				ksum += prevrank[j] / (jCidOutStake / linkStake)
-			}
-
-			// 17/20 = 0.85 = d
-			rank[cid] = (ksum+innerProductOverSize)/20*17 + tOverSize
-		}(i, inLinksForI)
-	}
-	wg.Wait()
-	return rank
-}
-
-func calculateChange(prevrank, rank []uint64) uint64 {
-
-	acc := uint64(0)
-	for i, pForI := range prevrank {
-		if pForI > rank[i] {
-			acc += pForI - rank[i]
-		} else {
-			acc += rank[i] - pForI
-		}
-	}
-
-	return acc
 }

cosmos/poc/app/rank/calculate_cpu.go

@@ -0,0 +1,95 @@
+package rank
+
+import (
+	. "github.com/cybercongress/cyberd/cosmos/poc/app/storage"
+	"sync"
+)
+
+const (
+	d         float64 = 0.85
+	tolerance float64 = 1e-3
+)
+
+func calculateRankCPU(data *InMemoryStorage) ([]float64, int) {
+
+	inLinks := data.GetInLinks()
+
+	size := data.GetCidsCount()
+	if size == 0 {
+		return []float64{}, 0
+	}
+
+	rank := make([]float64, size)
+	defaultRank := (1.0 - d) / float64(size)
+	danglingNodesSize := uint64(0)
+
+	for i := range rank {
+		rank[i] = defaultRank
+		if len(inLinks[CidNumber(i)]) == 0 {
+			danglingNodesSize++
+		}
+	}
+
+	innerProductOverSize := defaultRank * (float64(danglingNodesSize) / float64(size))
+	defaultRankWithCorrection := float64(d*innerProductOverSize) + defaultRank
+
+	change := tolerance + 1
+
+	steps := 0
+	prevrank := make([]float64, 0)
+	prevrank = append(prevrank, rank...)
+	for change > tolerance {
+		rank = step(defaultRankWithCorrection, prevrank, data)
+		change = calculateChange(prevrank, rank)
+		prevrank = rank
+		steps++
+	}
+
+	return rank, steps
+}
+
+func step(defaultRankWithCorrection float64, prevrank []float64, data *InMemoryStorage) []float64 {
+
+	rank := append(make([]float64, 0, len(prevrank)), prevrank...)
+
+	var wg sync.WaitGroup
+	wg.Add(len(data.GetInLinks()))
+
+	for i, inLinksForI := range data.GetInLinks() {
+
+		go func(cid CidNumber, inLinks CidLinks) {
+			defer wg.Done()
+			ksum := float64(0)
+
+			//todo dependent on range iterator order, that non-deterministic
+			for j := range inLinks {
+				linkStake := data.GetOverallLinkStake(CidNumber(j), CidNumber(cid))
+				jCidOutStake := data.GetOverallOutLinksStake(CidNumber(j))
+				weight := float64(linkStake) / float64(jCidOutStake)
+				ksum = float64(prevrank[j]*weight) + ksum //force no-fma here by explicit conversion
+			}
+
+			rank[cid] = float64(ksum*d) + defaultRankWithCorrection //force no-fma here by explicit conversion
+		}(i, inLinksForI)
+	}
+	wg.Wait()
+	return rank
+}
+
+func calculateChange(prevrank, rank []float64) float64 {
+
+	maxDiff := 0.0
+	diff := 0.0
+	for i, pForI := range prevrank {
+		if pForI > rank[i] {
+			diff = pForI - rank[i]
+		} else {
+			diff = rank[i] - pForI
+		}
+		if diff > maxDiff {
+			maxDiff = diff
+		}
+	}
+
+	return maxDiff
+}