Serializing a small computation to trade-off space for time.

Also updating sbt script.

cassovary-core/src/main/scala/com/twitter/cassovary/graph/SharedArrayBasedDirectedGraph.scala

@@ -227,18 +227,14 @@ object SharedArrayBasedDirectedGraph {
         }
         shardsInfo foreach { shardInfo =>
           shardInfo.idsMapped = new Array[Int](shardInfo.numIdsMapped)
-          shardInfo.nextFreeEdgeIndex.set(0) //reusing a variable
+          shardInfo.numIdsMapped = 0 //reusing this variable
         }
-        val futures = nodeCollection.iterator.grouped(1 + numNodes / parallelismLimit).toSeq map { ids =>
-          futurePool {
-            var shard: PerShardInfo = shardsInfo(0)
-            ids foreach { id =>
-              shard = shardsInfo(EdgeShards.hash(id))
-              shard.idsMapped(shard.nextFreeEdgeIndex.getAndIncrement()) = id
-            }
-          }
+        // fill out the array per shard by binning nodeCollection's ids into each shard
+        nodeCollection foreach { id =>
+          val shard = shardsInfo(EdgeShards.hash(id))
+          shard.idsMapped(shard.numIdsMapped) = id
+          shard.numIdsMapped += 1
         }
-        Future.join(futures)
       }
 
       // do function f(shardNum, id, idIndex) for all nodes, divided into EdgeShards.numShards futures
@@ -327,8 +323,8 @@ object SharedArrayBasedDirectedGraph {
 
       // main set of steps to build incoming edges in the graph
       log.info("Now building the reverse direction representation")
+      partitionNodeIdsPerShard()
       for {
-        _ <- partitionNodeIdsPerShard()
         _ <- findInEdgesSizes()
         _ <- findInShardSizes()
         sharedInEdges = instantiateSharedArray(reverseShardsInfo)

project/Build.scala

@@ -6,7 +6,7 @@ import xerial.sbt.Sonatype._
 
 object Cassovary extends Build {
 
-  val CassovaryLibraryVersion = "6.3.0"
+  val CassovaryLibraryVersion = "6.3.1"
 
   val utilVersion = "6.28.0"
   val fastUtilsDependency = "it.unimi.dsi" % "fastutil" % "7.0.7"

sbt

@@ -5,16 +5,20 @@ root=$(
   /bin/pwd
 )
 
-sbtjar=sbt-launch.jar
 sbtver=0.13.9
-SBTURL=http://dl.bintray.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/$sbtver/$sbtjar
+
+rawsbtjar=sbt-launch.jar
+sbtdir=$HOME/.sbt/launchers/$sbtver
+sbtjar=$sbtdir/$rawsbtjar
+SBTURL=http://dl.bintray.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/$sbtver/$rawsbtjar
 JARMD5=767d963ed266459aa8bf32184599786d
 
 if [ ! -f $sbtjar ]; then
-  echo "downloading $sbtjar" 1>&2
+  echo "downloading $rawsbtjar" 1>&2
   curl --location --remote-name $SBTURL
-  echo "copying to $HOME/.sbt/launchers/$sbtver/$sbtjar"
-  cp $sbtjar $HOME/.sbt/launchers/$sbtver/
+  echo "copying to $sbtdir "
+  mkdir -p $sbtdir
+  cp $rawsbtjar $sbtdir
 fi
 
 test -f $sbtjar || exit 1