Use chunked parallelelization in verify_and_store

src/block_add.rs

@@ -231,45 +231,49 @@ fn verify_and_store_transactions(store: &mut Store, block: &Block) -> BlockResul
         return Err(BlockError::BlockTooLarge);
     }
 
-    // hash all transactions (in parallel)
-    let hashes: Vec<Hash32Buf> = if block.txs.len() > PARALLEL_HASHING_THRESHOLD {
-        block.txs.par_iter().map(|tx| {
-            Hash32Buf::double_sha256(tx.to_raw())
-        }).collect()
-    }
-    else // or sequential
-    {
-        block.txs.iter().map(|tx| {
-            Hash32Buf::double_sha256(tx.to_raw())
-        }).collect()
-    };
+    let chunks: Vec<_> = block.txs.par_chunks(PARALLEL_HASHING_THRESHOLD).map(|chunk_tx| {
 
+        let len = chunk_tx.len();
 
-    // here we verify and store
-    let mut records: Vec<Record> = Vec::new();
-    for (n,tx) in block.txs.iter().enumerate() {
+        let mut hashes:  Vec<Hash32Buf> = Vec::with_capacity(len); // accurate
+        let mut records: Vec<Record>    = Vec::with_capacity(len * 3); // estimate
 
-        let hash = hashes[n].as_ref();
+        let ref mut tx_index = &mut store.tx_index.clone();
+        let ref mut tx_store = &mut store.transactions.clone();
 
-        let res = tx.verify_and_store(&mut store.tx_index, &mut store.transactions, hash).unwrap();
+        for tx in chunk_tx {
 
-        // AlreadyExists and VerifiedAndStored are both ok here
-        let ptr = match res {
-            transaction::TransactionOk::AlreadyExists(ptr) => ptr,
-            transaction::TransactionOk::VerifiedAndStored(ptr) => ptr
-        };
+            let hash = Hash32Buf::double_sha256(tx.to_raw());
+            hashes.push(hash);
 
-        records.push(Record::new_transaction(ptr));
-        records.append(&mut tx.get_output_records(store));
+            let res = tx.verify_and_store(tx_index, tx_store, hash.as_ref()).unwrap();
+
+            // AlreadyExists and VerifiedAndStored are both ok here
+            let ptr = match res {
+                transaction::TransactionOk::AlreadyExists(ptr) => ptr,
+                transaction::TransactionOk::VerifiedAndStored(ptr) => ptr
+            };
+
+            records.push(Record::new_transaction(ptr));
+        }
+        (hashes,records)
+    }).collect();
+
+
+    // split
+    let (hashes,records): (Vec<_>, Vec<_>) = chunks.into_iter().unzip();
+
+    // flatten
+    let hashes  = hashes.into_iter().flat_map(|x| x);
+    let records = records.into_iter().flat_map(|x| x);
 
-    }
 
     // check merkle roots
-    let calculated_merkle_root = merkle_tree::get_merkle_root(hashes);
+    let calculated_merkle_root = merkle_tree::get_merkle_root(hashes.collect());
     block.verify_merkle_root(calculated_merkle_root.as_ref()).unwrap();
 
 
-    Ok(records)
+    Ok(records.collect())
 }
 
 

src/store/README.md

@@ -15,22 +15,28 @@ Each file starts with a 16 byte header:
 (both in native endian)
 8 bytes reserved.
 
-In normal operation the files are append-only and writes and reads occur lock-free. Writes will first increase the 
+In most operation the files are append-only and writes and reads occur lock-free. Writes will first increase the 
 write-pointer (using atomic compare-and-swap) and then write the record at the location of the previous write pointer.
 
-Data in the files can be identified with 46-bit pointers (16-bit fileno and 30-bit filepos) [(src)](fileptr.rs)
+Data in the files can be identified with pointer that implements the *flatfileptr* Trait. 
 
-## Block_Content
+This provides a file number (max 16-bit) and a file offset (max 32 bit).
 
 
-Transactions and blockheaders are stored in flatfiles `block_content/bc-XXXX`  [(src)](block_content.rs) 
-Transanctions are prefixed with a 4-byte length and written in network format. Blockheaders are not length-prefixed, and also stored in network format.
+## Block Content
 
 
-## Hash_Index
+Transactions and blockheaders are stored in flatfiles `transactions/tx-XXXX` 
+and `headers/bh-0000`. 
 
-The hash index is used to lookup fileptrs from hashes of both blocks and transactions. 
-It is stored in flat_files `hash_index/ht-XXXX` [(src)](hash_index.rs). The first 64mb of the flatfileset is 
+Both are prefixed with a 4-byte length and written in network format. 
+Blockheaders are not length-prefixed, and also stored in network format.
+
+
+## Hash Index
+
+Hashes of blocks and transactions are looked in two hash-indexes [(src)](hash_index.rs). 
+It is stored in flat_files `hash_index/ht-XXXX` . The first 64mb of the flatfileset is 
 the root node; it is a hash-table to resolve the first 24-bits of a hash. This points to a append-only unbalanced 
 binary tree.