driver

#!/usr/bin/tclsh

# Copyright IBM Corp. 2016. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# 		 http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set usage {
Usage: driver ?...args..? [... peers ...]

This is a test driver for the 'counters' chaincode. In a typical usage this
driver:
    o Deploys 1 or more instances of the chaincode to the network
    o Creates 1 or more counter arrays of various sizes in the blockchain state
    o Creates 1 or more clients to drive transactions. Each client:
          - Drives a fixed number of transactions into the network
          - Chaincodes, arrays and peers to target are generated randomly
          - However, in the end all arrays get the same count
    o Finally, validates the final state of all counters

By default, the peer network is defined by the $BUSYWORK_HOME/network file.
However, if one or more peers are provided on the command line, those explicit
peers will be used instead, and the -port option then specifies the default
REST API port to use if an explicit peer does not include an explicit port.
Note that the list of explict peers *must* contain all peers in the
network. This is required for certain important consistency and correctness
checks. Use -targetPeers to limit the peers that are actually targeted by the
driver if reducing the number of targeted peers is important to the test.

Optional parameters. Default values are given after the colon:

-h | -help | --help : None

    Print this usage message and exit normally.

-home <busywork_home> : See below

    This argument can be used to name a non-default BUSYWORK_HOME directory.
    network on the command line. If not defined here then BUSYWORK_HOME is
    taken from the environmnent, or if not present there, defaults to
    ~/.busywork.

-remote | -local : -local

    You must specify -remote if the peer network is running remotely, that is,
    not on the same machine as the client driver.

-targetPeers <n> : All

    Limit the run to targeting at most the first <n> peers.

-security | -noSecurity : -noSecurity

    If -security is selected then the network is assumed to be a secure
    network, and the users are selected from the test_user* defined in the
    membersrvc.yaml file. This flag is only required if the peers are listed
    explicitly. Normally the security mode is obtained from the
    $BUSYWORK/network configuration, and specifying either version of this
    flag is considered an error in this case.

-targetUsers <n> : All

    If -security is selected and this parameter is specified, then at most
    this many of the test users will be logged in and used randomly to send
    transactions. By default all test users are used.

-port <n> : 7050

    The default REST API port. Only used if the peer specification does not
    include a port specifiaction.

-chaincodes <n> : 1

    Deploy <n> instances of the 'counters' chaincode. Chaincode IDs are

        <ccPrefix><m> : <m> = 1,...,<n>

    The default <ccPrefix> is 'cc', so by default the chaincoces are named
    'cc0', 'cc1', ..., etc. Use the -ccPrefix option to change the prefix.

    Specify -noDeploy to avoid deployment and the -deployWait. This parameter
    is effectively ignored in -noDeploy mode.

-ccPrefix <prefix> : cc

    This option can be used to change the chaincode name prefix. This should only
    be required in special cases to avoid ambiguities when a peer network is
    being driven my multiple independent sets of client drivers.

-deploy | -noDeploy : -deploy

    It is assumed by default that the target network is a 'fresh' network, and
    all chaincodes are deployed by the driver.

    -noDeploy can be specified to run multiple tests against a network without
    redeploying the chaincodes. Note that the arrays are always (re-)created
    each time the driver is run.

    If -noDeploy is specified, then the driver will examine the
    $BUSYWORK_HOME/chaincodes file to determine the number and names of the
    'counters' chaincodes that have been previously deployed, and they will
    all be targeted.

    If running with -security in -noDeploy mode it is assumed that all users
    are already logged in. Therefore one should not change the number of users
    to avoid problems.

-deployWait : 5m

    The maximum amount of time to wait for chaincode deployment to be
    registered in the blockchain. It may still take some time after this
    before the chaincodes are actually running.

-chaincodeWait : 5m

    Once the chaincodes are registered as deployed, wait up to this amount of
    time for a successful "ping" of all of the chaincodes. A chaincode that
    "pings" successfully is up and running and ready to go.

-arrays <n> : 1

    Create <n> arrays in each chaincode. Array names are

        a<m> : <m> = 1,...,<n>

-size <size> | *<n> : *1

    This keyword specifies the size(s) of the counter arrays. If the parameter
    parses as a simple integer, then that <size> applies to all arrays. If the
    form is *<n>, then for each of the <m> = 1,...,N arrays, the size of array
    <m> is <m> * <n> counters. By defualt (*1), each array has as many counters
    as its array index, i.e., the arrays are of increasingly larger sizes.

-rest : -rest

    This is the protocol used for the chaincode transactions. Currently
    only REST API calls are supported.

-clients <n> : 1

    The number of simultaneous clients to activate. Even with 1 client,
    clients run as subprocesses that are fork()-ed from the driver.

-transactions <n> : 1

    This is the number of transactions that are sent to each array in each
    chaincode by each client. At the end of a successful run the count value
    of each counter will be <clients> * <n>.

-peerBurst <n> : 1

    Specifies the transaction burst length for each peer. The client first
    prepares and then "bursts" (at most) this many transactions as fast as
    possible each time it addresses a peer.

-netBurst <n> : 1

    Specifies the number of Peer bursts that are made each time the network is
    addressed. If -interlock is specified, interlock occurs after the network
    burst is completed, not after each peer burst.

    Note that -peerBurst and -netBurst do not apply to chaincode deployment or
    array creation. Chaincodes are deployed sequentially as fast as
    possible. All arrays for a chaincode are created in a single transaction,
    and all 'create' transactions are generated as quickly as possible by the
    base driver.

-txDelay <duration> : 0
-peerDelay <duration> : 0
-netDelay <duration> : 0

    These are delays added after each transaction (-txDelay), each burst to a
    peer (-peerDelay) and each burst to the network (-netDelay).

-interlock | -noInterlock : -noInterlock

    If -interlock is specified, then each client waits after each network
    burst until all transactions of the last burst have been committed to the
    blockchain. By default (-noInterlock) the client simply sends all required
    transactions as fast as possible, subject to the bursting specifications.

    Note that array creation is currently always interlocked to avoid ordering
    issues.

-finalInterlock : N/A

    Note: Not implemented yet

    If -finalInterlock is specified, then only one interlock is done by each
    client, after all transactions have been presented. It is illegal to
    specify -interlock and -finalInterlock simultaneously (the -interlock
    would have already interlocked all transactions). The -interlockTimeout
    and -interlockProgress parameters also apply to the final interlock.

-interlockTimeout <duration> : 60s

    In -interlock mode, if interlock does not occur within this timeout then
    an error is signalled. Set this value to -1 to disable the timeout.

    Note that since array creation is always interlocked this timeout will
    always be in effect, at least for array creation.

-interlockProgress | -interlockAbsolute : -interlockAbsoute

    Note: Not implemented yet

    In -interlockAbsolute mode the script fails if any interlock is not
    achieved within the -interlockTimeout. In -interlockProgress mode, the
    driver continues to attempt interlock as long as at least 1 transaction
    completes within each timeout period.

-convergeWait <duration> : 10s
-convergePoll <duration> : 1s

    If the run is in -interlock mode, then the run will not finish until at
    least one peer has recorded every transaction. However due to delays and
    state transfer, there may be lagging peers that have not finished yet. In
    this case we do "strong reads" of the block height of every node, and do
    not start checking until there is 100% consensus on the block
    height. These checks must converge within the -convergeWait time. The
    network is polled (and the current status printed) every -convergePoll
    interval.

-finalWait <duration> : 10s

    Always wait this duration at the end of the run before checking the final
    status for correctness. Wired things can still happen even if the
    blockchains appear to be converged.

-keepLog | -noKeepLog : -noKeepLog

    By default, the fabricLogger log file used for interlock is deleted at the
    end of the test. Use -keepLog to keep it from being deleted.

-force | -noForce : -noForce

    The idea of -force is to force the test to continue, even in the event of
    errors. This allows the final consistency checking and the -checkAgreement
    option to report even if the run would have failed due to an error from
    the client, or from the consistency check. Even with -force, the script
    will fail with an abnormal exit code if any errors were encountered.

-watchdogPoll <duration> : 5s

    By default a busywork 'networkStatus' process is started and runs in the
    background to monitor the network every 5s and kill the driver if network
    nodes die. If the -watchdogPoll option is negative, or if -remote is
    specified, or if -force is specified, or if explicit peers are specified,
    then the watchdog process is not started. (The final restriction could be
    removed with some work on the 'networkStatus' script.)

-pprofClient <args> : No profiling

    If the -pprofClient option is provided, then a busywork 'pprofClient'
    process will be started immediately after the driver threads are started,
    where the <args> are passed to the pprofClient process. See the
    pprofClient script for details on parameters to pprofClient. A typical use
    might look something like

        -pprofClient "-wait 5 0 profile?seconds=60"

    which waits 5 seconds for the network to "warm up" before starting the
    profile, then collects a 60-second CPU profile of peer 0 into the
    $BUSYWORK_HOME directory.

-checkAgreement <extraAargs> : Extra arguments (possibly empty) for
                               checkAgreement

    If selected, there are at least 2 peers, and NOOPS consensus is not being
    used, then the ../bin/checkAgreement procedure will be run at the end of
    the run to ensure that all peers agree on the structure of the
    Blockchain. If there are no extra arguments, specify as the empty
    string. The logs generated by checkAgreement will be written to the
    BUSYWORK_HOME directory unless overridden in the <extraargs>.

-noops : N/A
-logUsingPeerN <n> : Log using all peers in round-robin order

    If the network is using NOOPS consensus and is explicitly defined by the
    -peers option, then you must also specify -noops in order for the checks
    to be configured correctly. If the network defaults to a busywork network
    then this flag is not required. This is required because NOOPS does not
    guarantee consistency of the blockchains on multiple peers.

    NOOPS also forces the option "-logUsingPeerN 0", that restricts the
    fabricLogger to only logging from the first peer. This option can also be
    given explictly, where the argument is a 0-based integer N, and only the
    Nth peer is used by the fabricLogger for interlock processing.

-retry : 0

    This parameter specifies the number of HTTP retries allowed for each
    INVOKE transacton. This is required due to poor performance/capacity of
    gRPC, which is reflected back as failures of INVOKE transactions. This
    parameter is also passed to the fabricLogger, since if HTTP invokes are
    failing the fabricLogger's REST API queries are likely to fail as well.

-args <args> : N/A

    This parameter is used to pass extra arguments to the 'counters' chaincode
    'parms' method when the chaincode is deployed. The driver always supplies
    the -id <name> parameter. See the documentation for the 'counters'
    chaincode for descriptions of the other arguments.

-timestamp | -noTimestamp : -noTimestamp

    If -timestamp is selected, the logs from this script will be timestamped.

-traceInterlock : N/A

    This is a debugging flag. If set, a timestamped trace of the interlock
    operations of the clients is produced. Clients "interlock" after each
    network burst if -interlock is selected, waiting until all transactions of
    the burst are seen as having been committed to the blockchain.

}

############################################################################
# Option Processing
############################################################################

lappend auto_path [file dirname [info script]]/../tcl

package require busywork

signal trap SIGINT {exit 1};    # Error exit on ^C or SIGINT

setLoggingPrefix driver

set options {
    {enum    {-h -help --help}     parms(help)              0 p_help}
    {key     -home                 parms(home)             {}}
    {bool    {-remote -local}      parms(remote)            0}
    {key     -targetPeers          parms(targetPeers)       0 p_targetPeers}
    {bool    {-security -noSecurity} parms(security)        0 p_security}
    {key     -targetUsers          parms(targetUsers)       0 p_targetUsers}
    {key     -port                 parms(port)              7050}
    {key     -chaincodes           parms(nChaincodes)       1}
    {key     -ccPrefix             parms(ccPrefix)         cc}
    {bool    {-deploy -noDeploy}   parms(deploy)            1}
    {key     -deployWait           parms(deployWait)        5m}
    {key     -chaincodeWait        parms(chaincodeWait)     5m}
    {key     -arrays               parms(nArrays)           1}
    {key     -size                 parms(size)              *1}
    {enum    {-rest}               parms(protocol)          -rest}
    {key     -clients              parms(clients)           1}
    {key     -transactions         parms(transactions)      1}
    {key     -peerBurst            parms(peerBurst)         1}
    {key     -netBurst             parms(netBurst)          1}
    {key     -txDelay              parms(txDelay)           0}
    {key     -peerDelay            parms(peerDelay)         0}
    {key     -netDelay             parms(netDelay)          0}
    {bool    {-interlock -noInterlock} parms(interlock)     0}
    {bool    -finalInterlock       parms(finalInterlock)    0}
    {key     -interlockTimeout     parms(interlockTimeout)  60s}
    {key     {-interlockProgress -interlockAbsolute} parms(interlockProgress) 0}
    {key     -convergeWait         parms(convergeWait)      10s}
    {key     -convergePoll         parms(convergePoll)      1s}
    {key     -finalWait            parms(finalWait)         10s}
    {bool    {-keepLog -noKeepLog} parms(keepLog)           0}
    {bool    {-force -noForce}     parms(force)             0}
    {key     -watchdogPoll         parms(watchdogPoll)      5s}
    {key     -pprofClient          parms(pprofClient)       {} p_pprofClient}
    {key     -checkAgreement       parms(checkAgreementArgs) {} p_checkAgreement}
    {bool    -noops                parms(noops)             0}
    {key     -logUsingPeerN        parms(logUsingPeerN)     {} p_logUsingPeerN}
    {key     -retry                parms(retry)             0}
    {key     -args                 parms(args)              {}}
    {bool    {-timestamp -noTimestamp} parms(timestamp)     0}
    {bool    -traceInterlock       parms(traceInterlock)    0}
}

mapKeywordArgs $argv $options parms(explicitPeers)

if {$p_help} {
    puts $usage
    exit 0
}

setLoggingLevel {} note
parms client driver

note {} "$argv0 $argv"

set BUSYWORK_HOME [busywork::home [parms home]]


# Start log timestamping if selected. Do quick consistency checks.

setLoggingTimestamp [parms timestamp]

if {[parms interlock] && [parms finalInterlock]} {
    errorExit \
        "It is illegal to specify -interlock and " \
        "-finalInterlock simultaneously"
}


# Handle implicit vs. explict peers. We need to keep the original list of peers
# for the -checkAgreement option, and their number to check that all
# chaincodes have been started. In implicit mode the security setup is defined
# in the network configuration. Same for NOOPS.

if {[null [parms explicitPeers]]} {
    if {[catch {busywork::networkToArray ::parms network.} msg]} {
        errorExit $msg
    }
    parms originalPeers [parms network.peer.restAddresses]
    parms noops [? {[parms network.consensus] eq "noops"} 1 0]
    if {$p_security} {
        errorExit \
            "The security mode is implicit in the busywork network. " \
            "The -security/-noSecurity flags are not allowed."
    } else {
        parms security [? {[parms network.security] eq "true"} 1 0]
    }
} else {
    parms originalPeers [addPortToHosts [parms explicitPeers] [parms port]]
}

parms totalPeers [llength [parms originalPeers]]

if {$p_targetPeers} {
    parms peers [firstn [parms originalPeers] [parms targetPeers]]
} else {
    parms peers [parms originalPeers]
}

parms nPeers [llength [parms peers]]
if {[parms nPeers] == 0} {
    errorExit "No peers were specified!"
}

note {} "List of targeted peers with REST API ports"
foreach peer [parms peers] {
    note {} "    $peer"
}

# Handle defaults and exclusions around NOOPS, then set up logging.

if {[parms noops]} {
    if {$p_checkAgreement} {
        warn {} "-checkAgreement can not be used with NOOPS networks: Ignoring"
        set p_checkAgreement 0
        parms checkAgreement {}
    }
    if {!$p_logUsingPeerN} {
        note {} "Forcing '-logUsingPeerN 0' for NOOPS"
        set p_logUsingPeerN 1
        parms logUsingPeerN 0
    }
}

if {$p_logUsingPeerN} {

    if {[catch {lindex [parms originalPeers] [parms logUsingPeerN]} peer] ||
        [null $peer]} {

        errorExit "Illegal value '[parms logUsingPeerN]' for -logUsingPeerN"
    }
    parms loggingPeers $peer

} else {

    parms loggingPeers [parms peers]
}

CircularList create peerList [parms peers]


# Compute the names and sizes of the arrays.

set names {}
for {set i 1} {$i <= [parms nArrays]} {incr i} {
    lappend names a$i
}
parms arrayNames $names
note {} "[llength [parms arrayNames]] array(s) will be defined per chaincode"

parms countPerArray [expr {[parms transactions] * [parms clients]}]

set sizes {}
if {[string is integer [parms size]]} {
    for {set i 0} {$i < [parms nArrays]} {incr i} {
        lappend sizes [parms size]
    }
} elseif {[string index [parms size] 0] eq "*"} {
    set multiplier [string range [parms size] 1 end]
    if {![string is integer $multiplier]} {
        errorExit "Illegal/unrecognized size specification -> '[parms size]'"
    }
    for {set i 1} {$i <= [parms nArrays]} {incr i} {
        lappend sizes [expr {$i * $multiplier}]
    }
} else {
    errorExit "Illegal/unrecognized size specification -> '[parms size]'"
}
parms arraySizes $sizes


# Normalize delays to milliseconds

parms txDelay [durationToMs [parms txDelay]]
parms peerDelay [durationToMs [parms peerDelay]]
parms netDelay [durationToMs [parms netDelay]]

############################################################################
# Setup
############################################################################

# Start a fabricLogger process, logging to a temporary file. Start a polling
# networkStatus process with a killer-callback (under most conditions).

set logger [::busywork::Logger new \
                -peers "[parms loggingPeers]" \
                [? [parms keepLog] -keepLog -noKeepLog] \
                -retry [parms retry] \
                [? [parms timestamp] -timestamp -noTimestamp] \
                [? [parms force] -noKillOnError -killOnError]]

parms watchdogPoll [durationToMs [parms watchdogPoll]]
if {[null [parms explicitPeers]] &&
    ![parms remote] && ![parms force] && ([parms watchdogPoll] >= 0)} {
    if {[catch \
             {exec [busywork::bin]/networkStatus \
                  -poll [parms watchdogPoll]ms -quiet \
                  -onError "kill SIGINT [pid]" &} \
             pid]} {
        errorExit "Error starting networkStatus : $pid"
    }
    killAtExit SIGINT $pid
}


############################################################################
# Security
############################################################################

# If security is enabled, compute the list of users that will be used and log
# them in. The current security implementation only allows a user to login to
# 1 peer. Thus we match users with peers. If there are fewer users than peers
# we have to reduce the number of peers we target. The users that are
# logged-in to each peer are stored in RandomBag-s and whenever we need to
# transact on a peer we pick a user. If we're not doing security then we also
# create a RandomBag for each peer, populated with the single NULL ({}) user.
# If we are running with -noDeploy, then it is assumed that the users are
# already logged in from the previous run.

if {[parms security]} {

    note {} "Security is enabled"

    busywork::usersAndPasswordsToArray ::parms security.
    set users [mapeach user [parms security.users] {
        if {[string match test_user* $user]} {
            return $user
        } else {
            continue
        }
    }]

    if {$p_targetUsers} {
        set users [firstn $users [parms targetUsers]]
    }

    parms users $users

    if {[llength [parms users]] < [llength [parms peers]]} {
        warn {} \
            "There are fewer users than peers; Reducing the number " \
            "of targeted peers to [llength [parms users]]"
        parms peers [firstn [parms peers] [llength [parms users]]]
    }

    set pos 0
    foreach peer [parms peers] {
        parms $peer,userBag \
            [RandomBag new \
                 [everyNth [parms users] [llength [parms peers]] $pos]]
        incr pos
    }

    note {} "Peer-to-user(s) mapping:"
    foreach peer [parms peers] {
        note {} "    $peer : [[parms $peer,userBag] list]"
    }

    if {[parms deploy]} {
        note {} "Logging users into peers"
        foreach peer [parms peers] {
            foreach user [[parms $peer,userBag] list] {
                ::fabric::caLogin \
                    $peer $user [parms security.user.$user.password]
            }
        }
    } else {
        note {} "-noDeploy mode : Assuming all users are logged in"
    }

} else {

    note {} "Security is disabled"

    foreach peer [parms peers] {
        parms $peer,userBag [RandomBag new {{}}]
    }
}


# Return a random user for a peer

proc peerUser {peer} {

    return [[parms $peer,userBag] pick]
}


############################################################################
# Deployment
############################################################################

# If we are deploying, we go to each peer in round-robin order to deploy the
# next of the chaincodes.

proc txSummary {} {
    parms totalTransactions \
        [expr {[parms countPerArray] * [parms nChaincodes] * [parms nArrays]}]

    note {} "[parms clients] clients will be activated"
    note {} "[parms countPerArray] transaction(s) will be issued to each array"
    note {} "[parms totalTransactions] total transaction(s) will be issued"
}

set chaincodePath "github.com/hyperledger/fabric/tools/busywork/counters"

if {[parms deploy]} {

    set chaincodeIDs {}
    for {set i 0} {$i < [parms nChaincodes]} {incr i} {
        lappend chaincodeIDs [parms ccPrefix]$i
    }
    parms chaincodeIDs $chaincodeIDs
    note {} "[parms nChaincodes] chaincode(s) will be deployed"

    txSummary

    note {} "Deploying chaincode(s)"

    set chaincodeNames {}
    foreach id [parms chaincodeIDs] {
        set peer [peerList next]
        set user [peerUser $peer]
        set args [concat [list -id $id] [parms args]]
        set name [::fabric::deploy $peer $user $chaincodePath parms $args]
        note {} "Deployed $id as $name"
        busywork::addChaincode $id $name $chaincodePath parms $args
        lappend chaincodeNames $name
    }
    parms chaincodeNames $chaincodeNames

    note {} \
        "Waiting up to [parms deployWait] for chaincode deployments " \
        "to be registered"
    set unmatched [$logger waitUUIDs \
                       deploy $chaincodeNames [parms deployWait]]
    if {$unmatched ne {}} {
        err {} "Chaincode deployment registration timed out; Aborting"
        err {} "Unmatched chaincode names below"
        foreach name $unmatched {
            err {} "    $name"
        }
        errorExit
    }

    proc ping {} {
        foreach name [parms chaincodeNames] {
            set peer [peerList next]
            set user [peerUser $peer]
            if {[catch {fabric::query $peer $user $name ping {} -1}]} {
                return 0
            }
        }
        return 1
    }

    note {} \
        "Waiting up to [parms chaincodeWait] for all chaincodes to be running"
    if {[waitFor [parms chaincodeWait] {ping} 1s]} {
        errorExit "Wait for chaincode startup timed out"
    }

} else {

    note {} "-noDeploy mode: Getting chaincode deployments from:"
    note {} "    $BUSYWORK_HOME/chaincodes"

    if {[catch {busywork::chaincodesToArray ::parms chaincodes.} why]} {
        err {} "Error accessing $BUSYWORK_HOME/chaincodes:"
        err {} "    $why"
        err {} "Have chaincodes been deployed to this network?"
        errorExit
    }
    set chaincodeIDs {}
    set chaincodeNames {}
    foreach id [parms chaincodes.ids] {
        if {[parms chaincodes.$id.path] eq $chaincodePath} {
            lappend chaincodeIDs $id
            lappend chaincodeNames [parms chaincodes.$id.name]
        }
    }
    parms chaincodeIDs $chaincodeIDs
    parms chaincodeNames $chaincodeNames
    parms nChaincodes [llength $chaincodeIDs]

    if {[parms nChaincodes] == 0} {
        err {} "No counters chaincodes were found in:"
        err {} "    $BUSYWORK_HOME/chaincodes"
        errorExit
    }
    note {} \
        "[parms nChaincodes] counters chaincode(s) " \
        "have been previously deployed"

    txSummary
}


############################################################################
# Creation
############################################################################

# Create the arrays. All arrays are created on a chaincode with a single
# transaction. We always interlock after array creation.

set args {}
foreach array [parms arrayNames] size [parms arraySizes] {
    lappend args $array $size
}

set uuids {}
foreach name [parms chaincodeNames] {
    set peer [peerList next]
    set user [peerUser $peer]
    lappend uuids \
        [::fabric::invoke $peer $user $name create $args]
}

note {} \
    "Waiting up to [parms interlockTimeout] for array creation to be registered"
set unmatched [$logger waitUUIDs invoke $uuids [parms interlockTimeout]]
if {$unmatched ne {}} {
    err err "Array creation timed out; Aborting"
    err err "Unmatched UUIDs below"
    foreach name $unmatched {
        err err "    $name"
    }
    errorExit
}


############################################################################
# Fork clients and drive
############################################################################

# The parameter arrayKeys is the cross product of chaincodeIDs and array names

set keys {}
foreach name [parms chaincodeNames] {
    foreach array [parms arrayNames] {
        lappend keys [list $name $array]
    }
}
parms arrayKeys $keys


# This local class is a RandomBag that will only return each element at most N
# times. This is how we guarantee exactly -transactions are sent to each
# chaincode/array.

oo::class create LimitedRandomBag {

    variable bag
    variable count

    constructor {i_list i_n} {

        set bag [RandomBag new $i_list]
        foreach elt $i_list {
            set count($elt) $i_n
        }
    }

    method pick {} {

        set x [$bag pick]
        if {![null $x]} {
            if {[incr count($x) -1] == 0} {
                $bag remove $x
            }
        }
        return $x
    }

    method pickList {i_n} {

        set l {}
        for {set i 0} {$i < $i_n} {incr i} {
            set x [my pick]
            if {[null $x]} break
            lappend l $x
        }
        return $l
    }
}


# The client sends transactions randomly until there are no more active
# arrays. We overlap preparing the next burst with the network processing of
# the previous burst.

proc clientRoutine {i_logger} {

    clearAtExitHandlers
    setLoggingPrefix client[format %3d [parms client]]

    $i_logger reset

    set peerBag [RandomBag new [parms peers]]
    set keyBag [LimitedRandomBag new [parms arrayKeys] [parms transactions]]
    set uuids {}
    set tx 0

    while {1} {

        set netBurst {}
        for {set i 0} {$i < [parms netBurst]} {incr i} {
            set burst [$keyBag pickList [parms peerBurst]]
            if {[null $burst]} break
            set peerBurst [list [$peerBag pick] $burst]
            lappend netBurst $peerBurst
        }

        if {[parms interlock] && ![null $uuids]} {

            if {[parms traceInterlock]} {
                set nTx [llength $uuids]
                set range "$tx - [expr {$tx + $nTx} - 1]"
                note {} "[timestamp]: Start Interlock: $range"
                incr tx $nTx
            }

            set unmatched \
                [$i_logger waitUUIDs invoke $uuids [parms interlockTimeout]]

            if {[parms traceInterlock]} {
                note {} "[timestamp]: End   Interlock: $range"
            }

            if {$unmatched ne {}} {
                err err "Interlock timed out; Aborting"
                err err "Unmatched UUIDs below"
                foreach uuid $unmatched {
                    err err "    $uuid"
                }
                errorExit
            }
        }

        if {[null $netBurst]} break

        set uuids {}
        foreach clause $netBurst {
            foreach {peer keys} $clause break
            foreach clause $keys {
                foreach {chaincodeName arrayName} $clause break
                debug {} \
                    "::fabric::invoke $peer $chaincodeName increment $arrayName"
                lappend uuids \
                    [::fabric::invoke \
                         $peer [peerUser $peer] \
                         $chaincodeName increment $arrayName \
                         [parms retry]]
                if {[parms txDelay]} {
                    after [parms txDelay]
                }
            }
            if {[parms peerDelay]} {
                after [parms peerDelay]
            }
        }
        if {[parms netDelay]} {
            after [parms netDelay]
        }
    }
}


# Fork clients. The parent continues the script once all clients have exited;
# clients run their driver routine and exit.

note {} "Spawning clients:"
set pids {}
for {set i 0} {$i < [parms clients]} {incr i} {
    flush stdout
    flush stderr
    rand32Seed [set seed [math::urandom32]]
    set pid [fork]
    switch $pid {
        -1 {
            errorExit "Fork failed"
        }
        0 {
            parms client $i
            clientRoutine $logger
            exit 0
        }
        default {
            lappend pids $pid
            killAtExit SIGINT $pid
            note {} "    Client $i is subprocess $pid with seed $seed"
        }
    }
}

# Start a 'pprofClient' if requested.

if {$p_pprofClient} {
    if {[catch \
             {eval exec [busywork::bin]/pprofClient [parms pprofClient] &} \
             pid]} {
        errorExit "Starting pprofClient failed : $pid"
    }
    parms pprofClientPID $pid
}

# Note that the 'errors' variable is initialized here. We carry on in the
# event of errors until the final agreement check.

note {} "Waiting (indefinitely) for subprocesses to complete"
set t [time {set errors [waitPIDs $pids]} 1]

if {!$errors} {
    set seconds [expr {[lindex $t 0] / 1e6}]
    set rate [format %.2f [expr {[parms totalTransactions] / $seconds}]]
    note {} \
        "Transaction rate : $rate per second " \
        "([parms totalTransactions] / $seconds)"
}

if {$p_pprofClient} {
    if {[null [wait -nohang [parms pprofClientPID]]]} {
        warn {} \
            "The pprofClient process (PID [parms pprofClientPID]) is " \
            " still running. The profile may be distorted by inactivity."
    }
}

if {$errors && ![parms force]} {
    errorExit "Aborting due to client errors"
}


# Finally, check the results for correctness. For the most accurate completed
# transaction rates you need to -interlock, as it is likely that not all
# invocations have been commited at this point if -interlock is not
# specified. We can't interlock NOOPS because the blockchains are not required
# to be identical.

if {[parms interlock] && ![parms noops]} {

    # For interlocked runs, do "strong reads" of the block height until
    # consensus is reached.

    proc strongReadBlockHeight {} {
        set originalHeights [mapeach peer [parms peers] {
            return [::fabric::height $peer]
        }]
        set heights [removeDuplicates $originalHeights]
        if {[llength $heights] != 1} {
            note {} "    Observed block heights: $originalHeights"
        }
        return [expr {[llength $heights] == 1}]
    }

    note {} \
        "Waiting up to [parms convergeWait] for block-height consensus, " \
        "polling every [parms convergePoll]"
    if {[waitFor \
             [parms convergeWait] strongReadBlockHeight [parms convergePoll]]} {
        if {![parms force]} {
            set errors 1
            errorExit "Aborting due to block-height consensus timeout"
        }
    }

}

# We always wait -finalWait before continuing with the checks. Wierd things
# can happen even after all TX have putatively been processed.

note {} "Waiting [parms finalWait] before collecting the final status"
after [durationToMs [parms finalWait]]


# Now we check the consistency of the counters. Due to state transfer it will
# not always be the case the every chaincode executes every transaction, but
# it must be the case that every chaincode has a state that makes it appear
# that it executed every transaction.

note {} "Checking consistency of the final state"
foreach peer [parms peers] {
    foreach name [parms chaincodeNames] id [parms chaincodeIDs] {
        debug {} "::fabric::query $peer $name status [parms arrayNames]"
        set status \
            [::fabric::query \
                 $peer [peerUser $peer] $name status [parms arrayNames]]
        foreach \
            {l1 l2 c1 c2} $status \
            arrayName [parms arrayNames] \
            size [parms arraySizes] {

                if {($l1 != $l2) || ($l1 != $size)} {
                    err {} \
                        "Peer $peer : Array length mismatch for " \
                            "chaincode $id, array $arrayName.\n" \
                            "Expected $size, Reported $l1 $l2"
                        set errors 1
                    }

                    if {$c1 != $c2} {
                        warn {} \
                            "Peer $peer: Expected/observed mismatch for " \
                            "chaincode $id, array $arrayName"
                        warn {} \
                            "Expected [parms countPerArray], reported $c1 $c2"
                    }

                    if {$c2 != [parms countPerArray]} {
                        err {} \
                            "Peer $peer: Final count mismatch for " \
                            "chaincode $id, array $arrayName"
                        err {} \
                            "Expected [parms countPerArray], Reported $c1 $c2"
                        set errors 1
                    }
                }
        }
}


# Check for peer agreement. Note that we do this even if the previous checks
# failed. There are interesting cases where the status may be inconsistent but
# the blockchains actually agree.

if {$p_checkAgreement} {

    note {} "Checking for agreement between the peers"
    set checkAgreement [file dirname [info script]]/../bin/checkAgreement
    if {[catch {eval execout -ignorestderr \
                    $checkAgreement \
                    -dir $env(BUSYWORK_HOME) [parms checkAgreementArgs] \
                    [parms originalPeers]}]} {
        err err "Peer agreement check failed"
        set errors 1
    }
}

if {$errors} {
    err err "Aborting due to errors, mismatches or disagreements above"
    exit 1
} else {
    note {} "Terminating normally"
    exit 0
}