Update to sync with the newest BoF version

scripts/common.sh

@@ -336,6 +336,46 @@ function loadDataForEvaluation {
     done
 }
 
+function loadDataForBoFFastTest {
+    expName=$1
+    scheme=$2
+    KVNumber=$3
+    keylength=$4
+    fieldlength=$5
+    simulatedClientNumber=${6:-"${defaultSimulatedClientNumber}"}
+    storageSavingTarget=${7:-"0.6"}
+    codingK=${8:-"4"}
+    extraFlag=${9:-}
+
+    # Gen params
+    dataSizeOnEachNode=$(dataSizeEstimation ${KVNumber} ${keylength} ${fieldlength})
+    initialDelayTime=$(initialDelayEstimation ${dataSizeOnEachNode} ${scheme})
+    waitFlushCompactionTime=$(waitFlushCompactionTimeEstimation ${dataSizeOnEachNode} ${scheme})
+    treeLevels=$(treeSizeEstimation ${KVNumber} ${keylength} ${fieldlength})
+
+    # Outpout params
+    echo "Start experiment to Loading ${scheme}, expName is ${expName}; KVNumber is ${KVNumber}, keylength is ${keylength}, fieldlength is ${fieldlength}, simulatedClientNumber is ${simulatedClientNumber}, storageSavingTarget is ${storageSavingTarget}, codingK is ${codingK}, extraFlag is ${extraFlag}. Estimation of data size on each node is ${dataSizeOnEachNode} GiB, initial delay is ${initialDelayTime}, flush and compaction wait time is ${waitFlushCompactionTime}."
+    echo ""
+    estimatedTotalRunningTime=$(((${waitFlushCompactionTime} + ${KVNumber} / 20000) / 60))
+    echo "The total running time is estimated to be ${estimatedTotalRunningTime} minutes."
+
+    # Load
+    load "${expName}" "${scheme}" "${KVNumber}" "${keylength}" "${fieldlength}" "${simulatedClientNumber}" "${storageSavingTarget}" "${codingK}"
+    flush "${expName}" "${scheme}" "${waitFlushCompactionTime}"
+
+    ## Collect load logs
+    for nodeIP in "${NodesList[@]}"; do
+        echo "Copy loading stats of loading for ${expName}-${targetScheme} back, current working on node ${nodeIP}"
+        if [ ! -d ${PathToELECTLog}/${targetScheme}/${ExpName}-Load-${nodeIP} ]; then
+            mkdir -p ${PathToELECTLog}/${targetScheme}/${ExpName}-Load-${nodeIP}
+        fi
+        scp -r ${UserName}@${nodeIP}:${PathToELECTLog} ${PathToELECTResultSummary}/${targetScheme}/${ExpName}-Load-KVNumber-${KVNumber}-KeySize-${keylength}-ValueSize-${fieldlength}-CodingK-${codingK}-Saving-${storageSavingTarget}-Node-${nodeIP}-Time-$(date +%s)
+        ssh ${UserName}@${nodeIP} "rm -rf '${PathToELECTLog}'; mkdir -p '${PathToELECTLog}'"
+        ssh ${UserName}@${OSSServerNode} "du -s --bytes ${PathToColdTier}/data > ${PathToELECTLog}/${ExpName}-${targetScheme}-KVNumber-${KVNumber}-KeySize-${keylength}-ValueSize-${fieldlength}-CodingK-${codingK}-Saving-${storageSavingTarget}-OSSStorage.log"
+        scp ${UserName}@${OSSServerNode}:${PathToELECTLog}/${ExpName}-${targetScheme}-KVNumber-${KVNumber}-KeySize-${keylength}-ValueSize-${fieldlength}-CodingK-${codingK}-Saving-${storageSavingTarget}-OSSStorage.log ${PathToELECTResultSummary}/${targetScheme}/${ExpName}-${targetScheme}-KVNumber-${KVNumber}-KeySize-${keylength}-ValueSize-${fieldlength}-CodingK-${codingK}-Saving-${storageSavingTarget}-OSSStorage.log
+    done
+}
+
 function doEvaluation {
     expName=$1
     scheme=$2
@@ -418,13 +458,18 @@ function recovery {
 }
 
 function copyRunningLogs {
+    # check whether the current node is the client node first
+    for localIP in "${localIPs[@]}"; do
+        if [ "${localIP}" == "${ClientNode}" ]; then
+            return 0
+        fi
+    done
     rsync -av --progress ${UserName}@${ClientNode}:"${PathToELECTResultSummary}/*" "${PathToELECTResultSummary}/"
-    # ssh ${UserName}@${ClientNode} "rm -rf '${PathToELECTResultSummary}'; mkdir -p '${PathToELECTResultSummary}'"
 }
 
 function cleanUp {
-     for nodeIP in "${NodesList[@]}"; do
+    for nodeIP in "${NodesList[@]}"; do
         echo "Delete old DB backup to free storage sapce, current working on node ${nodeIP}"
         ssh ${UserName}@${nodeIP} "rm -rf '${PathToELECTExpDBBackup}'; mkdir -p '${PathToELECTExpDBBackup}'"
     done
-}
+}

scripts/exp/Exp-simple.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+. /etc/profile
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
+source "${SCRIPT_DIR}/../common.sh"
+# Exp-Simple: YCSB core workloads, 3-way replication, (6,4) encoding, 60% target storage saving, 0.3M KV load.
+
+ExpName="Exp-simple"
+schemes=("elect")
+KVNumber=240000
+keyLength=24
+valueLength=1000
+simulatedClientNumber=${defaultSimulatedClientNumber}
+dataSizeEstimation=$(echo "scale=2; $KVNumber * ($keyLength + $valueLength) / 1024 / 1024 / 1024 * 3 * 1.75" | bc)
+
+# Setup hosts
+setupNodeInfo ./hosts.ini
+# Run Exp
+for scheme in "${schemes[@]}"; do
+    echo "Start experiment of ${scheme}"
+    # Load data for evaluation
+    loadDataForBoFFastTest "${ExpName}" "${scheme}" "${KVNumber}" "${keyLength}" "${valueLength}" "${simulatedClientNumber}"
+done
+
+# Generate the summarized results
+if [ -f "${PathToScripts}/exp/${ExpName}.log" ]; then
+    rm -rf "${PathToScripts}/exp/${ExpName}.log"
+fi
+# output storage usage
+echo "The storage overhead results:" >>${PathToScripts}/exp/${ExpName}.log
+echo "The estimated storage overhead (with 3-way replication, include metadata and Cassandra Logs): ${dataSizeEstimation} GiB" >>${PathToScripts}/exp/${ExpName}.log
+for scheme in "${schemes[@]}"; do
+    echo "Storage usage of ${scheme}" >>${PathToScripts}/exp/${ExpName}.log
+    ${PathToScripts}/count/fetchStorage.sh "${ExpName}" "${scheme}" "${KVNumber}" "${keyLength}" "${valueLength}" >>${PathToScripts}/exp/${ExpName}.log
+done
+cat ${PathToScripts}/exp/${ExpName}.log

scripts/exp/Exp0-simpleOverall.sh

@@ -2,7 +2,7 @@
 . /etc/profile
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
 source "${SCRIPT_DIR}/../common.sh"
-# Exp2: YCSB core workloads, 3-way replication, (6,4) encoding, 60% target storage saving, 10M KV + 1M OP.
+# Exp0: YCSB core workloads, 3-way replication, (6,4) encoding, 60% target storage saving, 10M KV + 1M OP.
 
 ExpName="Exp0-simpleOverall"
 schemes=("elect" "cassandra")
@@ -34,7 +34,6 @@ for scheme in "${schemes[@]}"; do
     startupFromBackup "${ExpName}" "${scheme}" "${KVNumber}" "${keyLength}" "${valueLength}"
     recovery "${ExpName}" "${scheme}" "${KVNumber}" "${RunningRoundNumber}"
 done
-
 copyRunningLogs
 
 # Generate the summarized results

scripts/playbook/playbook-fail.yaml

@@ -14,7 +14,7 @@
       when: "'FAILED' in command_result.stderr"
     - name: Wait until killed server is found
       pause:
-        seconds: 90
+        seconds: 60
 
 - name: Stop CPU&Memory Monitor
   hosts: elect_failure

scripts/playbook/playbook-load.yaml

@@ -40,7 +40,7 @@
       when: "'FAILED' in command_result.stderr"
     - name: Wait until all KVS nodes are ready
       pause:
-        seconds: 90
+        seconds: 60
 
 - name: Setup log levels
   hosts: elect_servers

scripts/playbook/playbook-startup.yaml

@@ -34,7 +34,7 @@
       when: "'FAILED' in command_result.stderr"
     - name: Wait until cold startup is done
       pause:
-        seconds: 90
+        seconds: 60
 
 - name: Setup log levels
   hosts: elect_servers

scripts/settings.sh

@@ -1,26 +1,28 @@
 #!/bin/bash
 . /etc/profile
 # Common params for all experiments
-NodesList=(192.168.0.21 192.168.0.22 192.168.0.23 192.168.0.25 192.168.0.26 192.168.0.28) # The IP addresses of the ELECT cluster nodes
-OSSServerNode="192.168.0.27" # The IP address of the OSS server node
-OSSServerPort=8000 # The port number of the OSS server node
-ClientNode="192.168.0.29" # The IP address of the client node (it can be the local node running the scripts)
-UserName="yjren" # The user name of all the previous nodes
-sudoPasswd="yjren" # The sudo password of all the previous nodes; we use this to install the required packages automatically; we assume all the nodes have the same user name. For the Chameleon cloud, please keep this as empty.
-PathToArtifact="/mnt/ssd/ELECT" # The path to the artifact folder; we assume all the nodes have the same path.
-PathToELECTExpDBBackup="/mnt/ssd/ELECTExpDBBackup" # The path to the backup folder for storing the loaded DB content; we assume all the nodes have the same path.
-PathToELECTLog="/mnt/ssd/ELECTLogs" # The path to the log folder for storing the experiment logs; we assume all the nodes have the same path.
-PathToELECTResultSummary="/mnt/ssd/ELECTResults" # The path to the result summary folder for storing the final experiment results; we assume all the nodes have the same path. 
+NodesList=(10.31.0.184 10.31.0.182 10.31.0.189 10.31.0.187 10.31.0.180 10.31.0.183) # The IP addresses of the ELECT cluster nodes
+OSSServerNode="10.31.0.185"                                                         # The IP address of the OSS server node
+OSSServerPort=8000                                                                  # The port number of the OSS server node
+ClientNode="10.31.0.186"                                                            # The IP address of the client node (it can be the local node running the scripts)
+UserName="cc"                                                                       # The user name of all the previous nodes
+sudoPasswd=""                                                                       # The sudo password of all the previous nodes; we use this to install the required packages automatically; we assume all the nodes have the same user name. For the Chameleon cloud, please keep this as empty.
+PathToSSHPrivateKeyFile="/home/${UserName}/.ssh/id_rsa"                             # The path to the cluster SSH keys (Only used for Chameleon Cloud instance created with CC-* series system image); we assume all the nodes have the same path.
+PathToArtifact="/home/${UserName}/elect"                                            # The path to the artifact folder; we assume all the nodes have the same path.
+PathToELECTExpDBBackup="/home/${UserName}/ELECTExpDBBackup"                         # The path to the backup folder for storing the loaded DB content; we assume all the nodes have the same path.
+PathToELECTLog="/home/${UserName}/ELECTLogs"                                        # The path to the log folder for storing the experiment logs; we assume all the nodes have the same path.
+PathToELECTResultSummary="/home/${UserName}/ELECTResults"                           # The path to the result summary folder for storing the final experiment results; we assume all the nodes have the same path.
+DiskUsageMonitorTargetDiskMountPoint="/"                                            # The mount point of the target disk for monitoring the disk usage
 
 PathToELECTPrototype="${PathToArtifact}/src/elect"
 PathToYCSB="${PathToArtifact}/scripts/ycsb"
 PathToScripts="${PathToArtifact}/scripts"
 PathToColdTier="${PathToArtifact}/src/coldTier"
 
 NodeNumber="${#NodesList[@]}"
-SSTableSize=4
+SSTableSize=4 # Default is 4, change to 1 in BoF
 LSMTreeFanOutRatio=10
-concurrentEC=64
+concurrentEC=$((64 * 4 / $SSTableSize))
 defaultSimulatedClientNumber=16
 
 NodesList=($(printf "%s\n" "${NodesList[@]}" | sort -V))
@@ -30,6 +32,7 @@ FullNodeList+=("${ClientNode}")
 
 # Variable to store the matching interface
 networkInterface=""
+localIPs=($(hostname -I))
 
 # Loop through each IP in the list
 for ip in "${FullNodeList[@]}"; do

scripts/setup.sh

@@ -6,8 +6,8 @@ setupMode=${1:-"partial"}
 # SSH key-free connection from control node to all nodes
 for nodeIP in "${NodesList[@]}" "${OSSServerNode}" "${ClientNode}"; do
     if [ ${UserName} == "cc" ]; then
+        cp ${PathToSSHPrivateKeyFile} ~/.ssh/
         ssh-keyscan -H ${nodeIP} >>~/.ssh/known_hosts
-        scp ~/.ssh/config cc@${nodeIP}:~/.ssh/
         scp ~/.ssh/id_rsa cc@${nodeIP}:~/.ssh/
     else
         echo "Set SSH key-free connection to node ${nodeIP}"
@@ -21,9 +21,12 @@ for nodeIP in "${NodesList[@]}" "${OSSServerNode}" "${ClientNode}"; do
 done
 
 for nodeIP in "${NodesList[@]}" "${OSSServerNode}" "${ClientNode}"; do
-    rsync -av --progress ${PathToArtifact} ${UserName}@${nodeIP}:/mnt/ssd/
+    echo "Sending artifacts to ${nodeIP}"
+    rsync -av --progress ${PathToArtifact} ${UserName}@${nodeIP}:${PathToArtifact} &
 done
 
+wait
+
 # Install packages
 if [ ${setupMode} == "full" ]; then
     if [ ! -d "${PathToELECTExpDBBackup}" ]; then
@@ -124,5 +127,7 @@ FullNodeList+=("${ClientNode}")
 
 for nodeIP in "${FullNodeList[@]}"; do
     echo "Set up node ${nodeIP}"
-    ssh ${UserName}@${nodeIP} "cd ${PathToScripts}; bash setupOnEachNode.sh ${setupMode}"
+    ssh ${UserName}@${nodeIP} "cd ${PathToScripts}; bash setupOnEachNode.sh ${setupMode}" &
 done
+
+wait # Wait until all nodes are done.

scripts/status/statsDB_DISK_NET.sh

@@ -23,7 +23,7 @@ function statsDisk_Network_DB {
     echo "Bytes sent: $TX_BYTES" >>$NET_OUTPUT_FILE
 
     # Mount point
-    MOUNT_POINT="/mnt/ssd"
+    MOUNT_POINT=${DiskUsageMonitorTargetDiskMountPoint}
 
     # File to store the results
     IO_OUTPUT_FILE="${PathToELECTLog}/${expName}_${workload}_${stage}_disk_io_total.txt"

src/elect/conf/elect.yaml

@@ -601,7 +601,7 @@ task_delay: 1
 stripe_update_frequency: 10
 max_send_sstables: 32
 cold_period: 1
-max_stripe_update_sstable: 2
+max_stripe_update_sstable: 4
 max_concurrent_download: 16
 user_name: elect
 concurrent_recovery: 16

src/elect/src/java/org/apache/cassandra/io/erasurecode/net/ECMetadata.java

@@ -430,43 +430,5 @@ public long serializedSize(ECMetadata t, int version) {
 
     }
 
-    // public static byte[] readBytesFromFile(String fileName) throws IOException
-    // {
-    // // String fileName = descriptor.filenameFor(Component.DATA);
-    // File file = new File(fileName);
-    // long fileLength = file.length();
-    // FileInputStream fileStream = new FileInputStream(fileName);
-    // byte[] buffer = new byte[(int)fileLength];
-    // int offset = 0;
-    // int numRead = 0;
-    // while (offset < buffer.length && (numRead = fileStream.read(buffer, offset,
-    // buffer.length - offset)) >= 0) {
-    // offset += numRead;
-    // }
-    // if (offset != buffer.length) {
-    // throw new IOException(String.format("Could not read %s, only read %d bytes",
-    // fileName, offset));
-    // }
-    // fileStream.close();
-    // logger.debug("ELECT-Debug: read file {} successfully!", fileName);
-    // return buffer;
-    // // return ByteBuffer.wrap(buffer);
-    // }
-
-    // public static Object byteArrayToObject(byte[] bytes) throws Exception {
-    // ByteArrayInputStream bis = new ByteArrayInputStream(bytes);
-    // ObjectInputStream ois = new ObjectInputStream(bis);
-    // Object obj = ois.readObject();
-    // bis.close();
-    // ois.close();
-    // return obj;
-    // }
-    public static void main(String[] args) throws Exception {
-        String ecMetadataFile = "./nb-627-big-EC.db";
-        byte[] ecMetadataInBytes = ECNetutils.readBytesFromFile(ecMetadataFile);
-        ECMetadataContent ecMetadata = (ECMetadataContent) ByteObjectConversion.byteArrayToObject(ecMetadataInBytes);
-        logger.debug("ELECT-Debug: [Debug recovery] read ecmetadata ({}) for old sstable ({})", ecMetadata.stripeId);
-
-    }
 
 }

src/elect/src/java/org/apache/cassandra/io/sstable/format/big/BigTableWriter.java

@@ -24,6 +24,7 @@
 import java.nio.ByteBuffer;
 import java.util.*;
 import java.util.stream.Stream;
+
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 
@@ -448,6 +449,7 @@ protected SSTableWriter.TransactionalProxy txnProxy() {
     class TransactionalProxy extends SSTableWriter.TransactionalProxy {
         // finalise our state on disk, including renaming
         private static final int contentSizeForGeneratingHash = 4 * 1024 * 1024;
+
         protected void doPrepare() {
             iwriter.prepareToCommit();
 
@@ -461,15 +463,16 @@ protected void doPrepare() {
                 // descriptor.filenameFor(Component.DATA));
                 dataFileSize = new File(descriptor.filenameFor(Component.DATA)).length();
                 long fileLength = dataFileSize;
-                if(dataFileSize > contentSizeForGeneratingHash) {
+                if (dataFileSize > contentSizeForGeneratingHash) {
                     fileLength = contentSizeForGeneratingHash;
                 }
 
-                if(fileLength < 0 || fileLength > Integer.MAX_VALUE) {
-                    throw new IllegalStateException(String.format("ELECT-ERROR: The file length of sstable (%s) is negative (%s)", descriptor.filenameFor(Component.DATA), dataFileSize));
+                if (fileLength < 0 || fileLength > Integer.MAX_VALUE) {
+                    throw new IllegalStateException(
+                            String.format("ELECT-ERROR: The file length of sstable (%s) is negative (%s)",
+                                    descriptor.filenameFor(Component.DATA), dataFileSize));
                 }
 
-
                 byte[] bytes = new byte[(int) fileLength];
                 dataFileReadForHash.readFully(bytes);
                 dataFileReadForHash.close();
@@ -493,9 +496,9 @@ protected void doPrepare() {
                         }
                         hashID = sb.toString();
                     }
-
-                    // logger.debug("[ELECT]: generated hash value for current SSTable is {}, hash length is {}, file length is ({})", 
-                    //                 hashID, hashID.length(), dataFileSize);
+                    logger.debug(
+                            "[ELECT]: generated hash value for current SSTable is {}, hash length is {}, file length is ({})",
+                            hashID, hashID.length(), dataFileSize);
                 } catch (NoSuchAlgorithmException e) {
                     hashID = null;
                     logger.error("[ELECT-ERROR]: Could not generated hash value for current SSTable = {}",

src/elect/src/java/org/apache/cassandra/io/sstable/metadata/StatsMetadata.java

@@ -180,8 +180,8 @@ public Boolean setupHashIDIfMissed(String fileName) {
                         }
                         this.hashID = sb.toString();
                     }
-
-                    logger.debug("[ELECT]: generated hash value for current SSTable is {}, hash length is {}", this.hashID, this.hashID.length());
+                    logger.debug("[ELECT]: generated hash value for current SSTable is {}, hash length is {}",
+                            this.hashID, this.hashID.length());
                 } catch (NoSuchAlgorithmException e) {
                     this.hashID = null;
                     // logger.debug("[ELECT]: Could not generated hash value for current SSTable =
@@ -198,15 +198,6 @@ public Boolean setupHashIDIfMissed(String fileName) {
         return true;
     }
 
-    // public Boolean setIsReplicationTransferredToErasureCodingFlag(boolean isReplicationTransferToErasureCoding) {
-    //     this.isReplicationTransferToErasureCoding = isReplicationTransferToErasureCoding;
-    //     return true;
-    // }
-
-    // public Boolean setIsDataMigrateToCloudFlag(boolean isDataMigrateToCloud) {
-    //     this.isDataMigrateToCloud = isDataMigrateToCloud;
-    //     return true;
-    // }
 
     /**
      * @param gcBefore gc time in seconds
@@ -262,7 +253,8 @@ public StatsMetadata mutateLevel(int newLevel) {
 
     public StatsMetadata setIsTransferredToErasureCoding(final boolean newIsReplicationTransferToErasureCoding) {
 
-        logger.debug("ELECT-Debug: setIsTransferredToErasureCoding is StatsMetadata ({})", newIsReplicationTransferToErasureCoding);
+        logger.debug("ELECT-Debug: setIsTransferredToErasureCoding is StatsMetadata ({})",
+                newIsReplicationTransferToErasureCoding);
 
         return new StatsMetadata(estimatedPartitionSize,
                 estimatedCellPerPartitionCount,