perfcollect

#!/bin/bash 

#############################################################################################################
# .NET Performance Data Collection Script
#############################################################################################################

#############################################################################################################
#
# ***** HOW TO USE THIS SCRIPT *****
#
# This script can be used to collect and view performance data collected with perf_event on Linux.

# It's job is to make it simple to collect performance traces.
#
# How to collect a performance trace:
# 1. Prior to starting the .NET process, set the environment variable COMPlus_PerfMapEnabled=1.
#    This tells the runtime to emit information that enables perf_event to resolve JIT-compiled code symbols.
# 
# 2. Setup your system to reproduce the performance issue you'd like to capture.  Data collection can be
#    started on already running processes.
# 
# 3. [Usage #1] use "collect" command
#    - Run this script: sudo ./perfcollect collect samplePerfTrace. This will start data collection.
#    - Let the repro run as long as you need to capture the performance problem.
#    - Hit CTRL+C to stop collection.
#
#    [Usage #2] use "start" and "stop" command
#    - Run this script: sudo ./perfcollect start samplePerfTrace. This will start data colletion.
#    - Let the repro run as long as you need to capture the performance problem.
#    - Run: sudo ./perfcollect stop samplePerfTrace. This will stop collection.
#
# 4. When collection is stopped, the script will create a trace.zip file matching the name specified on the
#    command line.  This file will contain the trace, JIT-compiled symbol information, and all debugging
#    symbols for binaries referenced by this trace that were available on the machine at collection time.
#
# How to view a performance trace:
# 1. Run this script: ./perfcollect view samplePerfTrace.trace.zip
#    This will extract the trace, place and register all symbol files and JIT-compiled symbol information
#    and start the perf_event viewer.  By default, you will be looking at a callee view - stacks are ordered
#    top down.  For a caller or bottom up view, specify '-graphtype caller'. 
#############################################################################################################

######################################
## FOR DEBUGGING ONLY
######################################
# set -x

######################################
## Collection Options
## NOTE: These values represent the collection defaults.
######################################

# Set when we parse command line arguments to determine if we should enable specific collection options.
collect_cpu=1
collect_threadTime=0
collect_offcpu=0
collect_system=0

######################################
## .NET Event Categories
######################################

# Separate GCCollectOnly list because our LTTng implementation doesn't set tracepoint verbosity.
# Once tracepoint verbosity is set, we can set verbosity and collapse this with DotNETRuntime_GCKeyword.
declare -a DotNETRuntime_GCKeyword_GCCollectOnly=(
    DotNETRuntime:GCStart
    DotNETRuntime:GCStart_V1
    DotNETRuntime:GCStart_V2
    DotNETRuntime:GCEnd
    DotNETRuntime:GCEnd_V1
    DotNETRuntime:GCRestartEEEnd
    DotNETRuntime:GCRestartEEEnd_V1
    DotNETRuntime:GCHeapStats
    DotNETRuntime:GCHeapStats_V1
    DotNETRuntime:GCCreateSegment
    DotNETRuntime:GCCreateSegment_V1
    DotNETRuntime:GCFreeSegment
    DotNETRuntime:GCFreeSegment_V1
    DotNETRuntime:GCRestartEEBegin
    DotNETRuntime:GCRestartEEBegin_V1
    DotNETRuntime:GCSuspendEEEnd
    DotNETRuntime:GCSuspendEEEnd_V1
    DotNETRuntime:GCSuspendEEBegin
    DotNETRuntime:GCSuspendEEBegin_V1
    DotNETRuntime:GCCreateConcurrentThread
    DotNETRuntime:GCTerminateConcurrentThread
    DotNETRuntime:GCFinalizersEnd
    DotNETRuntime:GCFinalizersEnd_V1
    DotNETRuntime:GCFinalizersBegin
    DotNETRuntime:GCFinalizersBegin_V1
    DotNETRuntime:GCMarkStackRoots
    DotNETRuntime:GCMarkFinalizeQueueRoots
    DotNETRuntime:GCMarkHandles
    DotNETRuntime:GCMarkOlderGenerationRoots
    DotNETRuntime:FinalizeObject
    DotNETRuntime:GCTriggered
    DotNETRuntime:IncreaseMemoryPressure
    DotNETRuntime:DecreaseMemoryPressure
    DotNETRuntime:GCMarkWithType
    DotNETRuntime:GCPerHeapHistory_V3
    DotNETRuntime:GCGlobalHeapHistory_V2
    DotNETRuntime:GCCreateConcurrentThread_V1
    DotNETRuntime:GCTerminateConcurrentThread_V1
)


declare -a DotNETRuntime_GCKeyword=(
    DotNETRuntime:GCStart
    DotNETRuntime:GCStart_V1
    DotNETRuntime:GCStart_V2
    DotNETRuntime:GCEnd
    DotNETRuntime:GCEnd_V1
    DotNETRuntime:GCRestartEEEnd
    DotNETRuntime:GCRestartEEEnd_V1
    DotNETRuntime:GCHeapStats
    DotNETRuntime:GCHeapStats_V1
    DotNETRuntime:GCCreateSegment
    DotNETRuntime:GCCreateSegment_V1
    DotNETRuntime:GCFreeSegment
    DotNETRuntime:GCFreeSegment_V1
    DotNETRuntime:GCRestartEEBegin
    DotNETRuntime:GCRestartEEBegin_V1
    DotNETRuntime:GCSuspendEEEnd
    DotNETRuntime:GCSuspendEEEnd_V1
    DotNETRuntime:GCSuspendEEBegin
    DotNETRuntime:GCSuspendEEBegin_V1
    DotNETRuntime:GCAllocationTick
    DotNETRuntime:GCAllocationTick_V1
    DotNETRuntime:GCAllocationTick_V2
    DotNETRuntime:GCAllocationTick_V3
    DotNETRuntime:GCCreateConcurrentThread
    DotNETRuntime:GCTerminateConcurrentThread
    DotNETRuntime:GCFinalizersEnd
    DotNETRuntime:GCFinalizersEnd_V1
    DotNETRuntime:GCFinalizersBegin
    DotNETRuntime:GCFinalizersBegin_V1
    DotNETRuntime:GCMarkStackRoots
    DotNETRuntime:GCMarkFinalizeQueueRoots
    DotNETRuntime:GCMarkHandles
    DotNETRuntime:GCMarkOlderGenerationRoots
    DotNETRuntime:FinalizeObject
    DotNETRuntime:PinObjectAtGCTime
    DotNETRuntime:GCTriggered
    DotNETRuntime:IncreaseMemoryPressure
    DotNETRuntime:DecreaseMemoryPressure
    DotNETRuntime:GCMarkWithType
    DotNETRuntime:GCJoin_V2
    DotNETRuntime:GCPerHeapHistory_V3
    DotNETRuntime:GCGlobalHeapHistory_V2
    DotNETRuntime:GCCreateConcurrentThread_V1
    DotNETRuntime:GCTerminateConcurrentThread_V1
)

declare -a DotNETRuntime_TypeKeyword=(
    DotNETRuntime:BulkType
)

declare -a DotNETRuntime_GCHeapDumpKeyword=(
    DotNETRuntime:GCBulkRootEdge
    DotNETRuntime:GCBulkRootConditionalWeakTableElementEdge
    DotNETRuntime:GCBulkNode
    DotNETRuntime:GCBulkEdge
    DotNETRuntime:GCBulkRootCCW
    DotNETRuntime:GCBulkRCW
    DotNETRuntime:GCBulkRootStaticVar
)

declare -a DotNETRuntime_GCSampledObjectAllocationHighKeyword=(
    DotNETRuntime:GCSampledObjectAllocationHigh
)

declare -a DotNETRuntime_GCHeapSurvivalAndMovementKeyword=(
    DotNETRuntime:GCBulkSurvivingObjectRanges
    DotNETRuntime:GCBulkMovedObjectRanges
    DotNETRuntime:GCGenerationRange
)

declare -a DotNETRuntime_GCHandleKeyword=(
    DotNETRuntime:SetGCHandle
    DotNETRuntime:DestroyGCHandle
)

declare -a DotNETRuntime_GCSampledObjectAllocationLowKeyword=(
    DotNETRuntime:GCSampledObjectAllocationLow
)

declare -a DotNETRuntime_ThreadingKeyword=(
    DotNETRuntime:WorkerThreadCreate
    DotNETRuntime:WorkerThreadTerminate
    DotNETRuntime:WorkerThreadRetire
    DotNETRuntime:WorkerThreadUnretire
    DotNETRuntime:IOThreadCreate
    DotNETRuntime:IOThreadCreate_V1
    DotNETRuntime:IOThreadTerminate
    DotNETRuntime:IOThreadTerminate_V1
    DotNETRuntime:IOThreadRetire
    DotNETRuntime:IOThreadRetire_V1
    DotNETRuntime:IOThreadUnretire
    DotNETRuntime:IOThreadUnretire_V1
    DotNETRuntime:ThreadpoolSuspensionSuspendThread
    DotNETRuntime:ThreadpoolSuspensionResumeThread
    DotNETRuntime:ThreadPoolWorkerThreadStart
    DotNETRuntime:ThreadPoolWorkerThreadStop
    DotNETRuntime:ThreadPoolWorkerThreadRetirementStart
    DotNETRuntime:ThreadPoolWorkerThreadRetirementStop
    DotNETRuntime:ThreadPoolWorkerThreadAdjustmentSample
    DotNETRuntime:ThreadPoolWorkerThreadAdjustmentAdjustment
    DotNETRuntime:ThreadPoolWorkerThreadAdjustmentStats
    DotNETRuntime:ThreadPoolWorkerThreadWait
    DotNETRuntime:ThreadPoolWorkingThreadCount
    DotNETRuntime:ThreadPoolIOPack
    DotNETRuntime:GCCreateConcurrentThread_V1
    DotNETRuntime:GCTerminateConcurrentThread_V1
)

declare -a DotNETRuntime_ThreadingKeyword_ThreadTransferKeyword=(
    DotNETRuntime:ThreadPoolEnqueue
    DotNETRuntime:ThreadPoolDequeue
    DotNETRuntime:ThreadPoolIOEnqueue
    DotNETRuntime:ThreadPoolIODequeue
    DotNETRuntime:ThreadCreating
    DotNETRuntime:ThreadRunning
)

declare -a DotNETRuntime_NoKeyword=(
    DotNETRuntime:ExceptionThrown
    DotNETRuntime:Contention
    DotNETRuntime:RuntimeInformationStart
    DotNETRuntime:EventSource
)

declare -a DotNETRuntime_ExceptionKeyword=(
    DotNETRuntime:ExceptionThrown_V1
    DotNETRuntime:ExceptionCatchStart
    DotNETRuntime:ExceptionCatchStop
    DotNETRuntime:ExceptionFinallyStart
    DotNETRuntime:ExceptionFinallyStop
    DotNETRuntime:ExceptionFilterStart
    DotNETRuntime:ExceptionFilterStop
    DotNETRuntime:ExceptionThrownStop
)

declare -a DotNETRuntime_ContentionKeyword=(
    DotNETRuntime:ContentionStart_V1
    DotNETRuntime:ContentionStop
    DotNETRuntime:ContentionStop_V1
)

declare -a DotNETRuntime_StackKeyword=(
    DotNETRuntime:CLRStackWalk
)

declare -a DotNETRuntime_AppDomainResourceManagementKeyword=(
    DotNETRuntime:AppDomainMemAllocated
    DotNETRuntime:AppDomainMemSurvived
)

declare -a DotNETRuntime_AppDomainResourceManagementKeyword_ThreadingKeyword=(
    DotNETRuntime:ThreadCreated
    DotNETRuntime:ThreadTerminated
    DotNETRuntime:ThreadDomainEnter
)

declare -a DotNETRuntime_InteropKeyword=(
    DotNETRuntime:ILStubGenerated
    DotNETRuntime:ILStubCacheHit
)

declare -a DotNETRuntime_JitKeyword_NGenKeyword=(
    DotNETRuntime:DCStartCompleteV2
    DotNETRuntime:DCEndCompleteV2
    DotNETRuntime:MethodDCStartV2
    DotNETRuntime:MethodDCEndV2
    DotNETRuntime:MethodDCStartVerboseV2
    DotNETRuntime:MethodDCEndVerboseV2
    DotNETRuntime:MethodLoad
    DotNETRuntime:MethodLoad_V1
    DotNETRuntime:MethodLoad_V2
    DotNETRuntime:MethodUnload
    DotNETRuntime:MethodUnload_V1
    DotNETRuntime:MethodUnload_V2
    DotNETRuntime:MethodLoadVerbose
    DotNETRuntime:MethodLoadVerbose_V1
    DotNETRuntime:MethodLoadVerbose_V2
    DotNETRuntime:MethodUnloadVerbose
    DotNETRuntime:MethodUnloadVerbose_V1
    DotNETRuntime:MethodUnloadVerbose_V2
)

declare -a DotNETRuntime_JitKeyword=(
    DotNETRuntime:MethodJittingStarted
    DotNETRuntime:MethodJittingStarted_V1
)

declare -a DotNETRuntime_JitTracingKeyword=(
    DotNETRuntime:MethodJitInliningSucceeded
    DotNETRuntime:MethodJitInliningFailed
    DotNETRuntime:MethodJitTailCallSucceeded
    DotNETRuntime:MethodJitTailCallFailed
)

declare -a DotNETRuntime_JittedMethodILToNativeMapKeyword=(
    DotNETRuntime:MethodILToNativeMap
)

declare -a DotNETRuntime_LoaderKeyword=(
    DotNETRuntime:ModuleDCStartV2
    DotNETRuntime:ModuleDCEndV2
    DotNETRuntime:DomainModuleLoad
    DotNETRuntime:DomainModuleLoad_V1
    DotNETRuntime:ModuleLoad
    DotNETRuntime:ModuleUnload
    DotNETRuntime:AssemblyLoad
    DotNETRuntime:AssemblyLoad_V1
    DotNETRuntime:AssemblyUnload
    DotNETRuntime:AssemblyUnload_V1
    DotNETRuntime:AppDomainLoad
    DotNETRuntime:AppDomainLoad_V1
    DotNETRuntime:AppDomainUnload
    DotNETRuntime:AppDomainUnload_V1
)

declare -a DotNETRuntime_LoaderKeyword=(
    DotNETRuntime:ModuleLoad_V1
    DotNETRuntime:ModuleLoad_V2
    DotNETRuntime:ModuleUnload_V1
    DotNETRuntime:ModuleUnload_V2
)

declare -a DotNETRuntime_SecurityKeyword=(
    DotNETRuntime:StrongNameVerificationStart
    DotNETRuntime:StrongNameVerificationStart_V1
    DotNETRuntime:StrongNameVerificationStop
    DotNETRuntime:StrongNameVerificationStop_V1
    DotNETRuntime:AuthenticodeVerificationStart
    DotNETRuntime:AuthenticodeVerificationStart_V1
    DotNETRuntime:AuthenticodeVerificationStop
    DotNETRuntime:AuthenticodeVerificationStop_V1
)

declare -a DotNETRuntime_DebuggerKeyword=(
    DotNETRuntime:DebugIPCEventStart
    DotNETRuntime:DebugIPCEventEnd
    DotNETRuntime:DebugExceptionProcessingStart
    DotNETRuntime:DebugExceptionProcessingEnd
)

declare -a DotNETRuntime_CodeSymbolsKeyword=(
    DotNETRuntime:CodeSymbols
)

declare -a DotNETRuntime_CompilationKeyword=(
    DotNETRuntime:TieredCompilationSettings
    DotNETRuntime:TieredCompilationPause
    DotNETRuntime:TieredCompilationResume
    DotNETRuntime:TieredCompilationBackgroundJitStart
    DotNETRuntime:TieredCompilationBackgroundJitStop
    DotNETRuntimeRundown:TieredCompilationSettingsDCStart
)

# Separate GCCollectOnly list because our LTTng implementation doesn't set tracepoint verbosity.
# Once tracepoint verbosity is set, we can set verbosity and collapse this with DotNETRuntimePrivate_GCPrivateKeyword.
declare -a DotNETRuntimePrivate_GCPrivateKeyword_GCCollectOnly=(
    DotNETRuntimePrivate:GCDecision
    DotNETRuntimePrivate:GCDecision_V1
    DotNETRuntimePrivate:GCSettings
    DotNETRuntimePrivate:GCSettings_V1
    DotNETRuntimePrivate:GCPerHeapHistory
    DotNETRuntimePrivate:GCPerHeapHistory_V1
    DotNETRuntimePrivate:GCGlobalHeapHistory
    DotNETRuntimePrivate:GCGlobalHeapHistory_V1
    DotNETRuntimePrivate:PrvGCMarkStackRoots
    DotNETRuntimePrivate:PrvGCMarkStackRoots_V1
    DotNETRuntimePrivate:PrvGCMarkFinalizeQueueRoots
    DotNETRuntimePrivate:PrvGCMarkFinalizeQueueRoots_V1
    DotNETRuntimePrivate:PrvGCMarkHandles
    DotNETRuntimePrivate:PrvGCMarkHandles_V1
    DotNETRuntimePrivate:PrvGCMarkCards
    DotNETRuntimePrivate:PrvGCMarkCards_V1
    DotNETRuntimePrivate:BGCBegin
    DotNETRuntimePrivate:BGC1stNonConEnd
    DotNETRuntimePrivate:BGC1stConEnd
    DotNETRuntimePrivate:BGC2ndNonConBegin
    DotNETRuntimePrivate:BGC2ndNonConEnd
    DotNETRuntimePrivate:BGC2ndConBegin
    DotNETRuntimePrivate:BGC2ndConEnd
    DotNETRuntimePrivate:BGCPlanEnd
    DotNETRuntimePrivate:BGCSweepEnd
    DotNETRuntimePrivate:BGCDrainMark
    DotNETRuntimePrivate:BGCRevisit
    DotNETRuntimePrivate:BGCOverflow
    DotNETRuntimePrivate:BGCAllocWaitBegin
    DotNETRuntimePrivate:BGCAllocWaitEnd
    DotNETRuntimePrivate:GCFullNotify
    DotNETRuntimePrivate:GCFullNotify_V1
    DotNETRuntimePrivate:PrvFinalizeObject
    DotNETRuntimePrivate:PinPlugAtGCTime
)

declare -a DotNETRuntimePrivate_GCPrivateKeyword=(
    DotNETRuntimePrivate:GCDecision
    DotNETRuntimePrivate:GCDecision_V1
    DotNETRuntimePrivate:GCSettings
    DotNETRuntimePrivate:GCSettings_V1
    DotNETRuntimePrivate:GCOptimized
    DotNETRuntimePrivate:GCOptimized_V1
    DotNETRuntimePrivate:GCPerHeapHistory
    DotNETRuntimePrivate:GCPerHeapHistory_V1
    DotNETRuntimePrivate:GCGlobalHeapHistory
    DotNETRuntimePrivate:GCGlobalHeapHistory_V1
    DotNETRuntimePrivate:GCJoin
    DotNETRuntimePrivate:GCJoin_V1
    DotNETRuntimePrivate:PrvGCMarkStackRoots
    DotNETRuntimePrivate:PrvGCMarkStackRoots_V1
    DotNETRuntimePrivate:PrvGCMarkFinalizeQueueRoots
    DotNETRuntimePrivate:PrvGCMarkFinalizeQueueRoots_V1
    DotNETRuntimePrivate:PrvGCMarkHandles
    DotNETRuntimePrivate:PrvGCMarkHandles_V1
    DotNETRuntimePrivate:PrvGCMarkCards
    DotNETRuntimePrivate:PrvGCMarkCards_V1
    DotNETRuntimePrivate:BGCBegin
    DotNETRuntimePrivate:BGC1stNonConEnd
    DotNETRuntimePrivate:BGC1stConEnd
    DotNETRuntimePrivate:BGC2ndNonConBegin
    DotNETRuntimePrivate:BGC2ndNonConEnd
    DotNETRuntimePrivate:BGC2ndConBegin
    DotNETRuntimePrivate:BGC2ndConEnd
    DotNETRuntimePrivate:BGCPlanEnd
    DotNETRuntimePrivate:BGCSweepEnd
    DotNETRuntimePrivate:BGCDrainMark
    DotNETRuntimePrivate:BGCRevisit
    DotNETRuntimePrivate:BGCOverflow
    DotNETRuntimePrivate:BGCAllocWaitBegin
    DotNETRuntimePrivate:BGCAllocWaitEnd
    DotNETRuntimePrivate:GCFullNotify
    DotNETRuntimePrivate:GCFullNotify_V1
    DotNETRuntimePrivate:PrvFinalizeObject
    DotNETRuntimePrivate:PinPlugAtGCTime
)

declare -a DotNETRuntimePrivate_StartupKeyword=(
    DotNETRuntimePrivate:EEStartupStart
    DotNETRuntimePrivate:EEStartupStart_V1
    DotNETRuntimePrivate:EEStartupEnd
    DotNETRuntimePrivate:EEStartupEnd_V1
    DotNETRuntimePrivate:EEConfigSetup
    DotNETRuntimePrivate:EEConfigSetup_V1
    DotNETRuntimePrivate:EEConfigSetupEnd
    DotNETRuntimePrivate:EEConfigSetupEnd_V1
    DotNETRuntimePrivate:LdSysBases
    DotNETRuntimePrivate:LdSysBases_V1
    DotNETRuntimePrivate:LdSysBasesEnd
    DotNETRuntimePrivate:LdSysBasesEnd_V1
    DotNETRuntimePrivate:ExecExe
    DotNETRuntimePrivate:ExecExe_V1
    DotNETRuntimePrivate:ExecExeEnd
    DotNETRuntimePrivate:ExecExeEnd_V1
    DotNETRuntimePrivate:Main
    DotNETRuntimePrivate:Main_V1
    DotNETRuntimePrivate:MainEnd
    DotNETRuntimePrivate:MainEnd_V1
    DotNETRuntimePrivate:ApplyPolicyStart
    DotNETRuntimePrivate:ApplyPolicyStart_V1
    DotNETRuntimePrivate:ApplyPolicyEnd
    DotNETRuntimePrivate:ApplyPolicyEnd_V1
    DotNETRuntimePrivate:LdLibShFolder
    DotNETRuntimePrivate:LdLibShFolder_V1
    DotNETRuntimePrivate:LdLibShFolderEnd
    DotNETRuntimePrivate:LdLibShFolderEnd_V1
    DotNETRuntimePrivate:PrestubWorker
    DotNETRuntimePrivate:PrestubWorker_V1
    DotNETRuntimePrivate:PrestubWorkerEnd
    DotNETRuntimePrivate:PrestubWorkerEnd_V1
    DotNETRuntimePrivate:GetInstallationStart
    DotNETRuntimePrivate:GetInstallationStart_V1
    DotNETRuntimePrivate:GetInstallationEnd
    DotNETRuntimePrivate:GetInstallationEnd_V1
    DotNETRuntimePrivate:OpenHModule
    DotNETRuntimePrivate:OpenHModule_V1
    DotNETRuntimePrivate:OpenHModuleEnd
    DotNETRuntimePrivate:OpenHModuleEnd_V1
    DotNETRuntimePrivate:ExplicitBindStart
    DotNETRuntimePrivate:ExplicitBindStart_V1
    DotNETRuntimePrivate:ExplicitBindEnd
    DotNETRuntimePrivate:ExplicitBindEnd_V1
    DotNETRuntimePrivate:ParseXml
    DotNETRuntimePrivate:ParseXml_V1
    DotNETRuntimePrivate:ParseXmlEnd
    DotNETRuntimePrivate:ParseXmlEnd_V1
    DotNETRuntimePrivate:InitDefaultDomain
    DotNETRuntimePrivate:InitDefaultDomain_V1
    DotNETRuntimePrivate:InitDefaultDomainEnd
    DotNETRuntimePrivate:InitDefaultDomainEnd_V1
    DotNETRuntimePrivate:InitSecurity
    DotNETRuntimePrivate:InitSecurity_V1
    DotNETRuntimePrivate:InitSecurityEnd
    DotNETRuntimePrivate:InitSecurityEnd_V1
    DotNETRuntimePrivate:AllowBindingRedirs
    DotNETRuntimePrivate:AllowBindingRedirs_V1
    DotNETRuntimePrivate:AllowBindingRedirsEnd
    DotNETRuntimePrivate:AllowBindingRedirsEnd_V1
    DotNETRuntimePrivate:EEConfigSync
    DotNETRuntimePrivate:EEConfigSync_V1
    DotNETRuntimePrivate:EEConfigSyncEnd
    DotNETRuntimePrivate:EEConfigSyncEnd_V1
    DotNETRuntimePrivate:FusionBinding
    DotNETRuntimePrivate:FusionBinding_V1
    DotNETRuntimePrivate:FusionBindingEnd
    DotNETRuntimePrivate:FusionBindingEnd_V1
    DotNETRuntimePrivate:LoaderCatchCall
    DotNETRuntimePrivate:LoaderCatchCall_V1
    DotNETRuntimePrivate:LoaderCatchCallEnd
    DotNETRuntimePrivate:LoaderCatchCallEnd_V1
    DotNETRuntimePrivate:FusionInit
    DotNETRuntimePrivate:FusionInit_V1
    DotNETRuntimePrivate:FusionInitEnd
    DotNETRuntimePrivate:FusionInitEnd_V1
    DotNETRuntimePrivate:FusionAppCtx
    DotNETRuntimePrivate:FusionAppCtx_V1
    DotNETRuntimePrivate:FusionAppCtxEnd
    DotNETRuntimePrivate:FusionAppCtxEnd_V1
    DotNETRuntimePrivate:Fusion2EE
    DotNETRuntimePrivate:Fusion2EE_V1
    DotNETRuntimePrivate:Fusion2EEEnd
    DotNETRuntimePrivate:Fusion2EEEnd_V1
    DotNETRuntimePrivate:SecurityCatchCall
    DotNETRuntimePrivate:SecurityCatchCall_V1
    DotNETRuntimePrivate:SecurityCatchCallEnd
    DotNETRuntimePrivate:SecurityCatchCallEnd_V1
)

declare -a DotNETRuntimePrivate_StackKeyword=(
    DotNETRuntimePrivate:CLRStackWalkPrivate
)

declare -a DotNETRuntimePrivate_PerfTrackPrivateKeyword=(
    DotNETRuntimePrivate:ModuleRangeLoadPrivate
)

declare -a DotNETRuntimePrivate_BindingKeyword=(
    DotNETRuntimePrivate:BindingPolicyPhaseStart
    DotNETRuntimePrivate:BindingPolicyPhaseEnd
    DotNETRuntimePrivate:BindingNgenPhaseStart
    DotNETRuntimePrivate:BindingNgenPhaseEnd
    DotNETRuntimePrivate:BindingLookupAndProbingPhaseStart
    DotNETRuntimePrivate:BindingLookupAndProbingPhaseEnd
    DotNETRuntimePrivate:LoaderPhaseStart
    DotNETRuntimePrivate:LoaderPhaseEnd
    DotNETRuntimePrivate:BindingPhaseStart
    DotNETRuntimePrivate:BindingPhaseEnd
    DotNETRuntimePrivate:BindingDownloadPhaseStart
    DotNETRuntimePrivate:BindingDownloadPhaseEnd
    DotNETRuntimePrivate:LoaderAssemblyInitPhaseStart
    DotNETRuntimePrivate:LoaderAssemblyInitPhaseEnd
    DotNETRuntimePrivate:LoaderMappingPhaseStart
    DotNETRuntimePrivate:LoaderMappingPhaseEnd
    DotNETRuntimePrivate:LoaderDeliverEventsPhaseStart
    DotNETRuntimePrivate:LoaderDeliverEventsPhaseEnd
    DotNETRuntimePrivate:FusionMessageEvent
    DotNETRuntimePrivate:FusionErrorCodeEvent
)

declare -a DotNETRuntimePrivate_SecurityPrivateKeyword=(
    DotNETRuntimePrivate:EvidenceGenerated
    DotNETRuntimePrivate:ModuleTransparencyComputationStart
    DotNETRuntimePrivate:ModuleTransparencyComputationEnd
    DotNETRuntimePrivate:TypeTransparencyComputationStart
    DotNETRuntimePrivate:TypeTransparencyComputationEnd
    DotNETRuntimePrivate:MethodTransparencyComputationStart
    DotNETRuntimePrivate:MethodTransparencyComputationEnd
    DotNETRuntimePrivate:FieldTransparencyComputationStart
    DotNETRuntimePrivate:FieldTransparencyComputationEnd
    DotNETRuntimePrivate:TokenTransparencyComputationStart
    DotNETRuntimePrivate:TokenTransparencyComputationEnd
)

declare -a DotNETRuntimePrivate_PrivateFusionKeyword=(
    DotNETRuntimePrivate:NgenBindEvent
)

declare -a DotNETRuntimePrivate_NoKeyword=(
    DotNETRuntimePrivate:FailFast
)

declare -a DotNETRuntimePrivate_InteropPrivateKeyword=(
    DotNETRuntimePrivate:CCWRefCountChange
)

declare -a DotNETRuntimePrivate_GCHandlePrivateKeyword=(
    DotNETRuntimePrivate:PrvSetGCHandle
    DotNETRuntimePrivate:PrvDestroyGCHandle
)

declare -a DotNETRuntimePrivate_LoaderHeapPrivateKeyword=(
    DotNETRuntimePrivate:AllocRequest
)

declare -a DotNETRuntimePrivate_MulticoreJitPrivateKeyword=(
    DotNETRuntimePrivate:MulticoreJit
    DotNETRuntimePrivate:MulticoreJitMethodCodeReturned
)

declare -a DotNETRuntimePrivate_DynamicTypeUsageKeyword=(
    DotNETRuntimePrivate:IInspectableRuntimeClassName
    DotNETRuntimePrivate:WinRTUnbox
    DotNETRuntimePrivate:CreateRCW
    DotNETRuntimePrivate:RCWVariance
    DotNETRuntimePrivate:RCWIEnumerableCasting
    DotNETRuntimePrivate:CreateCCW
    DotNETRuntimePrivate:CCWVariance
    DotNETRuntimePrivate:ObjectVariantMarshallingToNative
    DotNETRuntimePrivate:GetTypeFromGUID
    DotNETRuntimePrivate:GetTypeFromProgID
    DotNETRuntimePrivate:ConvertToCallbackEtw
    DotNETRuntimePrivate:BeginCreateManagedReference
    DotNETRuntimePrivate:EndCreateManagedReference
    DotNETRuntimePrivate:ObjectVariantMarshallingToManaged
)

declare -a EventSource=(
    DotNETRuntime:EventSource
)

declare -a LTTng_Kernel_ProcessLifetimeKeyword=(
    sched_process_exec
    sched_process_exit
)


######################################
## Global Variables
######################################

# Install without waiting for standard input
forceInstall=0

# Declare an array of events to collect.
declare -a eventsToCollect

# Use Perf_Event
usePerf=1

# Use LTTng
useLTTng=1

# LTTng Installed
lttngInstalled=0

# Collect hardware events
collect_HWevents=0

# Set to 1 when the CTRLC_Handler gets invoked.
handlerInvoked=0

# Log file
declare logFile
logFilePrefix='/tmp/perfcollect'
logEnabled=0

# Collect info to pass between processes
collectInfoFile=$(dirname `mktemp -u`)/'perfcollect.sessioninfo'

######################################
## Logging Functions
######################################
LogAppend()
{
    if (( $logEnabled == 1 ))
    then
        echo $* >> $logFile
    fi
}

RunSilent()
{
    if (( $logEnabled == 1 ))
    then
        echo "Running \"$*\"" >> $logFile
        $* >> $logFile 2>&1
        echo "" >> $logFile
    else
        $* > /dev/null 2>&1
    fi
}

RunSilentBackground()
{
    if (( $logEnabled == 1 ))
    then
        echo "Running \"$*\"" >> $logFile
        $* >> $logFile 2>&1  & sleep 1
        echo "" >> $logFile
    else
        $* > /dev/null 2>&1  & sleep 1
    fi

    # When handler is invoked, kill the process.
    pid=$!
    for (( ; ; ))
    do
        if [ "$handlerInvoked" == "1" ]
        then
            kill -INT $pid
            break;
        else
            sleep 1
        fi	
    done

    # Wait for the process to exit.
    wait $pid
}

InitializeLog()
{
    # Pick the log file name.
    logFile="$logFilePrefix.log"
    while [ -f $logFile ];
    do
        logFile="$logFilePrefix.$RANDOM.log"
    done

    # Mark the log as enabled.
    logEnabled=1

    # Start the log
    date=`date`
    echo "Log started at ${date}" > $logFile
    echo '' >> $logFile

    # The system information.
    LogAppend 'Machine info: '  `uname -a`
    if [ "$perfcmd" != "" ]
    then
        LogAppend 'perf version:'   `$perfcmd --version 2>&1`
    fi
    if [ "$lttngcmd" != "" ]
    then
        LogAppend 'LTTng version: ' `$lttngcmd --version`
    fi
    LogAppend
}

CloseLog()
{
    LogAppend "END LOG FILE"
    LogAppend "NOTE: It is normal for the log file to end right before the trace is actually compressed.  This occurs because the log file is part of the archive, and thus can't be written anymore."

    # The log itself doesn't need to be closed,
    # but we need to tell the script not to log anymore.
    logEnabled=0
}


######################################
## Helper Functions
######################################

##
# Console text color modification helpers.
##
RedText()
{
    tput=`GetCommandFullPath "tput"`
    if [ "$tput" != "" ]
    then
    	$tput setaf 1
    fi
}

GreenText()
{
    tput=`GetCommandFullPath "tput"`
    if [ "$tput" != "" ]
    then
        $tput setaf 2
    fi
}

BlueText()
{
    tput=`GetCommandFullPath "tput"`
    if [ "$tput" != "" ]
    then
        $tput setaf 6
    fi
}
YellowText()
{
    tput=`GetCommandFullPath "tput"`
    if [ "$tput" != "" ]
    then
        $tput setaf 3
    fi
}

ResetText()
{
    tput=`GetCommandFullPath "tput"`
    if [ "$tput" != "" ]
    then
        $tput sgr0
    fi
}

# $1 == Status message
WriteStatus()
{
    LogAppend $*
    BlueText
    echo $1
    ResetText
}

# $1 == Message.
WriteWarning()
{
    LogAppend $*
    YellowText
    echo $1
    ResetText
}

# $1 == Message.
FatalError()
{
    RedText
    echo "ERROR: $1"
    ResetText
    PrintUsage
    exit 1
}

EnsureRoot()
{
    # Warn non-root users.
    if [ `whoami` != "root" ]
    then
        RedText
        echo "This script must be run as root."
        ResetText
        exit 1;
    fi
}

######################################
# Command Discovery
######################################
DiscoverCommands()
{
    perfcmd=`GetCommandFullPath "perf"`
    if [ "$(IsDebian)" == "1" ]
    then
        # Test perf to see if it successfully runs or fails because it doesn't match the kernel version.
        $perfcmd --version > /dev/null 2>&1
        if [ "$?" == "1" ]
        then
            perfoutput=$($perfcmd 2>&1)
            if [ $? -eq 1 ]
            then
                perfMarker="perf_"
                if [[ "$perfoutput" == *"$perfMarker"* ]]
                then
                    foundWorkingPerf=0
                    WriteWarning "Perf is installed, but does not exactly match the version of the running kernel."
                    WriteWarning "This is often OK, and we'll try to workaround this."
                    WriteStatus "Attempting to find a working copy of perf."
                    # Attempt to find an existing version of perf to use.
                    # Order the search by newest directory first.
                    baseDir="/usr/bin"
                    searchPath="$baseDir/perf_*"
                    for fileName in $(ls -d --sort=time $searchPath)
                    do
                        $($fileName > /dev/null 2>&1)
                        if [ $? -eq 1 ]
                        then
                            # If $? == 1, then use this copy of perf.
                            perfcmd=$fileName
                            foundWorkingPerf=1
                            break;
                        fi
                    done
                    if [ $foundWorkingPerf -eq 0 ]
                    then
                        FatalError "Unable to find a working copy of perf.  Try re-installing via ./perfcollect install."
                    fi
                    WriteStatus "...FINISHED"
                fi
            fi
        fi
    fi

    # Check to see if perf is installed, but doesn't exactly match the current kernel version.
    # This happens when the kernel gets upgraded, or when running inside of a container where the
    # host and container OS versions don't match.
    perfoutput=$($perfcmd 2>&1)
    if [ $? -eq 2 ]
    then
        # Check the beginning of the output to see if it matches the kernel warning.
        warningText="WARNING: perf not found for kernel"
        if [[ "$perfoutput" == "$warningText"* ]]
        then
            foundWorkingPerf=0
            WriteWarning "Perf is installed, but does not exactly match the version of the running kernel."
            WriteWarning "This is often OK, and we'll try to workaround this."
            WriteStatus "Attempting to find a working copy of perf."
            # Attempt to find an existing version of perf to use.
            # Order the search by newest directory first.
            baseDir="/usr/lib/linux-tools"
            searchPath="$baseDir/*/"
            for dirName in $(ls -d --sort=time $searchPath)
            do
                candidatePerfPath="$dirName""perf"
                $($candidatePerfPath > /dev/null 2>&1)
                if [ $? -eq 1 ]
                then
                    # If $? == 1, then use this copy of perf.
                    perfcmd=$candidatePerfPath
                    foundWorkingPerf=1
                    break;
                fi
            done
            if [ $foundWorkingPerf -eq 0 ]
            then
                FatalError "Unable to find a working copy of perf.  Try re-installing via ./perfcollect install."
            fi
            WriteStatus "...FINISHED"
        fi
    fi

    lttngcmd=`GetCommandFullPath "lttng"`
    zipcmd=`GetCommandFullPath "zip"`
    unzipcmd=`GetCommandFullPath "unzip"`
}

GetCommandFullPath()
{
    echo `command -v $1`
}

######################################
# Prerequisite Installation
######################################

IsMariner()
{
    local mariner=0
    if [ -f /etc/lsb-release ]
    then
        local flavor=`cat /etc/lsb-release | grep DISTRIB_ID`
        if [ "$flavor" == "DISTRIB_ID=\"Mariner\"" ]
        then
            mariner=1
        fi
    fi

    echo $mariner
}

InstallPerf_Mariner()
{
    # Disallow non-root users.
    EnsureRoot

    # Install perf
    tdnf install -y kernel-tools

    # Install zip and unzip
    tdnf install -y zip unzip
}

IsAlpine()
{
    local alpine=0
    local apk=`GetCommandFullPath "apk"`
    if [ "$apk" != "" ]
    then
        alpine=1
    fi

    echo $alpine
}

InstallPerf_Alpine()
{
    # Disallow non-root users.
    EnsureRoot

    # Install perf
    apk add perf --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community

    # Install zip and unzip
    apk add zip unzip
}

IsRHEL()
{
    local rhel=0
    if [ -f /etc/redhat-release ]
    then
        rhel=1
    fi

    echo $rhel
}

InstallPerf_RHEL()
{
    # Disallow non-root users.
    EnsureRoot

    # Install perf
    yum install perf zip unzip
}

IsDebian()
{
    local debian=0
    local uname=`uname -a`
    if [[ $uname =~ .*Debian.* ]]
    then
        debian=1
    elif [ -f /etc/debian_version ]
    then
        debian=1
    fi

    echo $debian
}

InstallPerf_Debian()
{
    # Disallow non-root users.
    EnsureRoot

    # Check for the existence of the linux-tools package.
    pkgName='linux-tools'
    pkgCount=`apt-cache search $pkgName | grep -c $pkgName`
    if [ "$pkgCount" == "0" ]
    then
        pkgName='linux-perf'
        pkgCount=`apt-cache search $pkgName | grep -c $pkgName`
        if [ "$pkgCount" == "0" ]
        then
            FatalError "Unable to find a perf package to install."
        fi
    fi

    # Install zip and perf.
    apt-get install -y zip binutils $pkgName
}

IsSUSE()
{
    local suse=0
    if [ -f /usr/bin/zypper ]
    then
        suse=1
    fi

    echo $suse
}

InstallPerf_SUSE()
{
    # Disallow non-root users.
    EnsureRoot

    # Install perf.
    zypper install perf zip unzip
}

IsUbuntu()
{
    local ubuntu=0
    if [ -f /etc/lsb-release ]
    then
        local flavor=`cat /etc/lsb-release | grep DISTRIB_ID`
        if [ "$flavor" == "DISTRIB_ID=Ubuntu" ]
        then
            ubuntu=1
        fi
    fi
        
    echo $ubuntu
}

InstallPerf_Ubuntu()
{
    # Disallow non-root users.
    EnsureRoot

    # Install packages.
    BlueText
    echo "Installing perf_event packages."
    ResetText

    # Handle Azure instances.
    release=`uname -r`
    if [[ "$release" == *"-azure" ]]
    then
        apt-get install -y linux-tools-azure zip software-properties-common
    else
        apt-get install -y linux-tools-common linux-tools-`uname -r` zip software-properties-common
    fi
}

InstallPerf()
{
    if [ "$(IsUbuntu)" == "1" ]
    then
        InstallPerf_Ubuntu
    elif [ "$(IsSUSE)" == "1" ]
    then
        InstallPerf_SUSE
    elif [ "$(IsDebian)" == "1" ]
    then
        InstallPerf_Debian
    elif [ "$(IsRHEL)" == "1" ]
    then
        InstallPerf_RHEL
    elif [ "$(IsAlpine)" == "1" ]
    then
        InstallPerf_Alpine
    elif [ "$(IsMariner)" == "1" ]
    then
        InstallPerf_Mariner
    else
        FatalError "Auto install unsupported for this distribution.  Install perf manually to continue."
    fi
}

InstallLTTng_RHEL()
{
    # Disallow non-root users.
    EnsureRoot

    local isRHEL7=0
    local isRHEL8=0
    if [ -e /etc/redhat-release ]; then
        local redhatRelease=$(</etc/redhat-release)
        if   [[ $redhatRelease == "CentOS Linux release 7."* || $redhatRelease == "Red Hat Enterprise Linux "*"release 7."* ]]; then
            isRHEL7=1
        elif [[ $redhatRelease == "CentOS Linux release 8."* || $redhatRelease == "Red Hat Enterprise Linux "*"release 8."* ]]; then
            isRHEL8=1
        fi
    fi

    if  [ "$isRHEL7" == "1" ] || [ "$isRHEL8" == "1" ]
    then
        packageRepo="https://packages.efficios.com/repo.files/EfficiOS-RHEL7-x86-64.repo"

        if [ "$forceInstall" != 1 ]
        then
            # Prompt for confirmation, since we need to add a new repository.
            BlueText
            echo "LTTng installation requires that a new package repo be added to your yum configuration."
            echo "The package repo url is: $packageRepo"
            echo ""
            read -p "Would you like to add the LTTng package repo to your YUM configuration? [Y/N]" resp
            ResetText
        fi

        # Make sure that wget is installed.
        BlueText
        echo "Installing wget.  Required to add package repo."
        ResetText
        yum install wget

        # Connect to the LTTng package repo.
        wget -P /etc/yum.repos.d/ $packageRepo

        # Import package signing key.
        rpmkeys --import https://packages.efficios.com/rhel/repo.key

        # Update the yum package database.
        yum updateinfo
    fi

    # Install LTTng
    yum install -y lttng-tools lttng-ust babeltrace
    if  [ "$isRHEL7" == "1" ]
    then
        yum install -y kmod-lttng-modules
    else
        YellowText
        echo "LTTng kernel package (kmod-lttng-modules) is not available on this platform."
        ResetText
    fi
}

InstallLTTng_Debian()
{
    # Disallow non-root users.
    EnsureRoot

    # Install LTTng
    apt-get install -y lttng-tools liblttng-ust-dev
}

InstallLTTng_SUSE()
{
    # Disallow non-root users.
    EnsureRoot

    # Package repo url
    packageRepo="http://download.opensuse.org/repositories/devel:/tools:/lttng/openSUSE_13.2/devel:tools:lttng.repo"

    if [ "$forceInstall" != 1 ]
    then
        # Prompt for confirmation, since we need to add a new repository.
        BlueText
        echo "LTTng installation requires that a new package repo be added to your zypper configuration."
        echo "The package repo url is: $packageRepo"
        echo ""
        read -p "Would you like to add the LTTng package repo to your zypper configuration? [Y/N]" resp
        ResetText
    fi
    if [ "$resp" == "Y" ] || [ "$resp" == "y" ] || [ "$forceInstall" == 1 ]
    then

        # Add package repo.
        BlueText
        echo "Adding LTTng repo and running zypper refresh."
        ResetText
        zypper addrepo $packageRepo
        zypper refresh

        # Install packages.
        BlueText
        echo "Installing LTTng packages."
        ResetText
        zypper install lttng-tools lttng-modules lttng-ust-devel
    fi
}

InstallLTTng_Ubuntu()
{
    # Disallow non-root users.
    EnsureRoot

    # Install packages.
    BlueText
    echo "Installing LTTng packages."
    ResetText

    distroVersion=$(lsb_release -r -s)
    majorVersion=${distroVersion%%.*}
    if (( $majorVersion < 22 )); then
        apt-get install -y lttng-tools lttng-modules-dkms liblttng-ust0
    else
        YellowText
        echo "You're using Ubuntu 22 or newer. LTTng version available for this version does not work with .NET. Use -force to force installation."
        ResetText

        if [ "$forceInstall" == 1 ]; then
            BlueText
            echo "Installing LTTng anyway."
            ResetText

            apt-get install -y lttng-tools lttng-modules-dkms liblttng-ust1
        fi
    fi
}

InstallLTTng()
{
    if command -v lttng &> /dev/null
    then
        lttngInstalled=1

        BlueText
        echo "LTTng already installed."
        ResetText
        return
    fi

    if [ "$(IsUbuntu)" == "1" ]
    then
        InstallLTTng_Ubuntu
    elif [ "$(IsSUSE)" == "1" ]
    then
        InstallLTTng_SUSE
    elif [ "$(IsDebian)" == "1" ]
    then
        InstallLTTng_Debian
    elif [ "$(IsRHEL)" == "1" ]
    then
        InstallLTTng_RHEL
    elif [ "$(IsMariner)" == "1" ]
    then
        echo "Collection of events from lttng-ust is not supported on Mariner for .NET workloads, so no lttng-ust events will be collected."
    elif [ "$(IsAlpine)" == "1" ]
    then
        echo "lttng-tools is not available on Alpine Linux, so no lttng-ust events will be collected."
    else
        FatalError "Auto install unsupported for this distribution.  Install lttng and lttng-ust packages manually."
    fi
}

SupportsAutoInstall()
{
    local supportsAutoInstall=0
    if [ "$(IsUbuntu)" == "1" ] || [ "$(IsSUSE)" == "1" ] || [ "$(IsMariner)" == "1" ]
    then
        supportsAutoInstall=1
    fi
    
    echo $supportsAutoInstall
}

EnsurePrereqsInstalled()
{
    # If perf is not installed, then bail, as it is currently required.
    if [ "$perfcmd" == "" ]
    then
        RedText
        echo "Perf not installed."
        if  [ "$(SupportsAutoInstall)" == "1" ]
        then
            echo "Run ./perfcollect install"
            echo "or install perf manually."
        else
            echo "Install perf to proceed."
        fi
        ResetText
        exit 1
    fi

    # Disable LTTng use if running on Alpine or Mariner.
    if [ "$(IsAlpine)" == "1" ] || [ "$(IsMariner)" == "1" ]
    then
        useLTTng=0
    fi

    # If LTTng is installed, consider using it.
    if [ "$lttngcmd" == "" ] && [ "$useLTTng" == "1" ]
    then
        RedText
        echo "LTTng not installed."
        if  [ "$(SupportsAutoInstall)" == "1" ]
        then
            echo "Run ./perfcollect install"
            echo "or install LTTng manually."
        else
            echo "Install LTTng to proceed."
        fi
        ResetText
        exit 1

    fi

    # If zip or unzip are not installing, then bail.
    if [ "$zipcmd" == "" ] || [ "$unzipcmd" == "" ]
    then
        RedText
        echo "Zip and unzip are not installed."
        if [ "$(SupportsAutoInstall)" == "1" ]
        then
            echo "Run ./perfcollect install"
            echo "or install zip and unzip manually."
        else
            echo "Install zip and unzip to proceed."
        fi
        ResetText
        exit 1
    fi
}

######################################
# Argument Processing
######################################
action=''
inputTraceName=''
collectionPid=''
processFilter=''
graphType=''
perfOpt=''
viewer='perf'
gcCollectOnly=''
gcOnly=''
gcWithHeap=''
events=''
rawevents=''
bufferSize=''

ProcessArguments()
{
    # Set the action
    action=$1

    # Actions with no arguments.
    if [ "$action" == "livetrace" ] || [ "$action" == "setenv" ]
    then
        return
    fi
    
    # Not enough arguments.
    if [ "$#" -le "1" ]
    then
        FatalError "Not enough arguments have been specified."
    fi

    # Validate action name.
    if [ "$action" != "collect" ] && [ "$action" != "view" ] \
    && [ "$action" != "start" ] && [ "$action" != "stop" ]
    then
        FatalError "Invalid action specified."
    fi

    # Set the data file.
    inputTraceName=$2
    if [ "$inputTraceName" == "" ] 
    then
        FatalError "Invalid trace name specified."
    fi

    # Process remaining arguments.
    # First copy the args into an array so that we can walk the array.
    args=( "$@" )
    for (( i=2; i<${#args[@]}; i++ ))
    do
        # Get the arg.
        local arg=${args[$i]}

        # Convert the arg to lower case.
        arg=`echo $arg | tr '[:upper:]' '[:lower:]'`

        # Get the arg value.
        if [ ${i+1} -lt $# ]
        then
            local value=${args[$i+1]}

            # Keep the cases of '-events' value to match keyword variables
            # Keep the cases of '-rawevents'
            if [ "-events" != "$arg" ] && [ "-rawevents" != "$arg" ]
            then
                # Convert the value to lower case.
                value=`echo $value | tr '[:upper:]' '[:lower:]'`
            fi

            # Use upper case for '-buffersize' as perf requires upper case.
            if [ "-buffersize" == "$arg" ]
            then
                # Convert the value to upper case.
                value=`echo $value | tr '[:lower:]' '[:upper:]'`
            fi
        fi

        # Match the arg to a known value.
        if [ "-pid" == "$arg" ]
        then
            collectionPid=$value
            i=$i+1
        elif [ "-processfilter" == "$arg" ]
        then
            processFilter=$value
            i=$i+1
        elif [ "-graphtype" == "$arg" ]
        then
            graphType=$value
            i=$i+1
        elif [ "-threadtime" == "$arg" ]
        then
            collect_threadTime=1
        elif [ "-offcpu" == "$arg" ]
        then
            # Perf doesn't support capturing cpu-clock events and sched events concurrently.
            collect_cpu=0
            collect_offcpu=1
        elif [ "-system" == "$arg" ]
        then
            collect_system=1
        elif [ "-hwevents" == "$arg" ]
        then
            collect_HWevents=1
        elif [ "-perfopt" == "$arg" ]
        then
            perfOpt=$value
            i=$i+1
        elif [ "-viewer" == "$arg" ]
        then
            viewer=$value
            i=$i+1

            # Validate the viewer.
            if [ "$viewer" != "perf" ] && [ "$viewer" != "lttng" ]
            then
                FatalError "Invalid viewer specified.  Valid values are 'perf' and 'lttng'."
            fi
        elif [ "-nolttng" == "$arg" ]
        then
            useLTTng=0
        elif [ "-noperf" == "$arg" ]
        then 
            usePerf=0
        elif [ "-gccollectonly" == "$arg" ]
        then
            gcCollectOnly=1
        elif [ "-gconly" == "$arg" ]
        then
            gcOnly=1
        elif [ "-gcwithheap" == "$arg" ]
        then
            gcWithHeap=1
        elif [ "-events" == "$arg" ]
        then
            events=$value
            i=$i+1
        elif [ "-rawevents" == "$arg" ]
        then
            rawevents=$value
            i=$i+1
        elif [ "-collectsec" == "$arg" ]
        then
            duration=$value
            i=$i+1
        elif [ "-buffersize" == "$arg" ]
        then
            bufferSize="--mmap-pages="$value
            i=$i+1
        else
            echo "Unknown arg ${arg}, ignored..."
        fi
    done
    
}



##
# LTTng collection
##
lttngSessionName=''
lttngTraceDir=''
CreateLTTngSession()
{
    if [ "$action" == "livetrace" ]
    then
        output=`$lttngcmd create --live`
    else
        output=`$lttngcmd create`
    fi

    lttngSessionName=`echo $output | grep -o "Session.*created." | sed 's/\(Session \| created.\)//g'`
    lttngTraceDir=`echo $output | grep -o "Traces.*" | sed 's/\(Traces will be written in \|\)//g' | sed 's/\(Traces will be output to \|\)//g'`
}

SetupLTTngSession()
{
    
    # Setup per-event context information.
    RunSilent "$lttngcmd add-context --userspace --type vpid"
    RunSilent "$lttngcmd add-context --userspace --type vtid"
    RunSilent "$lttngcmd add-context --userspace --type procname"
    RunSilent "$lttngcmd add-context --kernel -t pid -t procname"

    if [ "$action" == "livetrace" ]
    then
        RunSilent "$lttngcmd enable-event --userspace --tracepoint DotNETRuntime:EventSource"
    elif [ "$gcCollectOnly" == "1" ]
    then
        usePerf=0
        EnableLTTngEvents ${DotNETRuntime_GCKeyword_GCCollectOnly[@]}
        EnableLTTngEvents ${DotNETRuntimePrivate_GCPrivateKeyword_GCCollectOnly[@]}
        EnableLTTngEvents ${DotNETRuntime_ExceptionKeyword[@]}
    elif [ "$gcOnly" == "1" ]
    then
        usePerf=0
        EnableLTTngEvents ${DotNETRuntime_GCKeyword[@]}
        EnableLTTngEvents ${DotNETRuntimePrivate_GCPrivateKeyword[@]}
        EnableLTTngEvents ${DotNETRuntime_JitKeyword[@]}
        EnableLTTngEvents ${DotNETRuntime_LoaderKeyword[@]}
        EnableLTTngEvents ${DotNETRuntime_ExceptionKeyword[@]}
    elif [ "$gcWithHeap" == "1" ]
    then
        usePerf=0
        EnableLTTngEvents ${DotNETRuntime_GCKeyword[@]}
        EnableLTTngEvents ${DotNETRuntime_GCHeapSurvivalAndMovementKeyword[@]}
    else
        if [ "$events" == "" ]
        then
            # Enable the default set of events.
            EnableLTTngEvents ${DotNETRuntime_ThreadingKeyword[@]}
            EnableLTTngEvents ${DotNETRuntime_ThreadingKeyword_ThreadTransferKeyword[@]}
            EnableLTTngEvents ${DotNETRuntime_NoKeyword[@]}
            EnableLTTngEvents ${DotNETRuntime_ExceptionKeyword[@]}
            EnableLTTngEvents ${DotNETRuntime_ContentionKeyword[@]}
            EnableLTTngEvents ${DotNETRuntime_JitKeyword_NGenKeyword[@]}
            EnableLTTngEvents ${DotNETRuntime_JitKeyword[@]}
            EnableLTTngEvents ${DotNETRuntime_LoaderKeyword[@]}
            EnableLTTngEvents ${DotNETRuntime_GCKeyword_GCCollectOnly[@]}
            EnableLTTngEvents ${DotNETRuntimePrivate_GCPrivateKeyword_GCCollectOnly[@]}
            EnableLTTngEvents ${DotNETRuntimePrivate_BindingKeyword[@]}
            EnableLTTngEvents ${DotNETRuntimePrivate_MulticoreJitPrivateKeyword[@]}
            EnableLTTngEvents ${DotNETRuntime_CompilationKeyword[@]}
        elif [ "$events" == "threading" ]
        then
            EnableLTTngEvents ${DotNETRuntime_ThreadingKeyword[@]}
        else
            # Enable other keywords
            events=(${events//","/" "})
            for event in ${events[@]}
            do
                if [ "${!event}" == "" ] || [ "$( echo `declare -p $event | grep -- -a` )" == "" ]
                then
                    echo Invalid keyword $event, skipped...
                    continue
                fi
                local -a 'keywords=("${'"$event"'[@]}")'
                if [ "${event#"LTTng_Kernel_"}" != "$event" ]
                then 
                    EnableLTTngKernelEvents ${keywords[@]}
                else
                    EnableLTTngEvents ${keywords[@]}
                fi
            done
        fi
        if [ "$rawevents" != "" ]
        then
            rawevents=(${rawevents//","/" "})
            for rawevent in ${rawevents[@]}
            do
                EnableLTTngEvents $rawevent
            done
        fi
    fi
}

DestroyLTTngSession()
{
    RunSilent "$lttngcmd destroy $lttngSessionName"
}

StartLTTngCollection()
{
    CreateLTTngSession
    SetupLTTngSession

    RunSilent "$lttngcmd start $lttngSessionName"
}

StopLTTngCollection()
{
    RunSilent "$lttngcmd stop $lttngSessionName"
    DestroyLTTngSession
}

# $@ == event names to be enabled
EnableLTTngEvents()
{
    args=( "$@" )
    for (( i=0; i<${#args[@]}; i++ ))
    do
        RunSilent "$lttngcmd enable-event -s $lttngSessionName -u --tracepoint ${args[$i]}"
    done
}

EnableLTTngKernelEvents()
{
    args=( "$@" )
    for (( i=0; i<${#args[@]}; i++ ))
    do 
        RunSilent "$lttngcmd enable-event -s $lttngSessionName -k ${args[$i]}"
    done
}

##
# Helper that processes collected data.
# This helper is called when the CTRL+C signal is handled.
##
ProcessCollectedData()
{
    # Make a new target directory.
    local traceSuffix=".trace"
    local traceName=$inputTraceName
    local directoryName=$traceName$traceSuffix
    mkdir $directoryName

    # Save LTTng trace files.
    if [ "$useLTTng" == "1" ]
    then
        LogAppend "Saving LTTng trace files."

        if [ -d $lttngTraceDir ]
        then
            RunSilent "mkdir lttngTrace"
            RunSilent "cp -r $lttngTraceDir lttngTrace"
        fi
    fi

    if [ "$usePerf" == 1 ]
    then
        # Get any perf-$pid.map files that were used by the
        # trace and store them alongside the trace.
        LogAppend "Saving perf.map files."
        RunSilent "$perfcmd buildid-list --with-hits"
        local mapFiles=`$perfcmd buildid-list --with-hits | grep /tmp/perf- | cut -d ' ' -f 2`
        for mapFile in $mapFiles
        do
            if [ -f $mapFile ]
            then
                LogAppend "Saving $mapFile"

                # Change permissions on the file before saving, as perf will need to access the file later
                # in this script when running perf script.
                RunSilent "chown root $mapFile"
                RunSilent "cp $mapFile ."
            else
                LogAppend "Skipping $mapFile.  Some managed symbols may not be resolvable, but trace is still valid."
            fi

            # Also check for jit-<pid>.dump files.
            # Convert to jit dump file name.
            local pid=`echo $mapFile | awk -F"-" '{print $NF}' | awk -F"." '{print $1}'`
            local path=`echo $mapFile | awk -F"/" '{OFS="/";NF--;print $0;}'`
            local jitDumpFile="$path/jit-$pid.dump"

            if [ -f $jitDumpFile ]
            then
                LogAppend "Saving $jitDumpFile"
                RunSilent "cp $jitDumpFile ."
            fi
        done

        WriteStatus "Resolving JIT and R2R symbols"

        originalFile="perf.data"
        inputFile="perf-jit.data"
        RunSilent $perfcmd inject --input $originalFile --jit --output $inputFile

        WriteStatus "...FINISHED"

        WriteStatus "Exporting perf.data file"

        outputDumpFile="perf.data.txt"

        LogAppend "Running $perfcmd script -i $inputFile --fields comm,pid,tid,cpu,time,period,event,ip,sym,dso,trace > $outputDumpFile"
        $perfcmd script -i $inputFile --fields comm,pid,tid,cpu,time,period,event,ip,sym,dso,trace > $outputDumpFile 2>>$logFile
        LogAppend

        # Try capturing without the cpu field which is unavailable in some containerized environments.
        if [ $? -ne 0 ]
        then
            LogAppend "Running $perfcmd script -i $inputFile --fields comm,pid,tid,time,period,event,ip,sym,dso,trace > $outputDumpFile"
            $perfcmd script -i $inputFile --fields comm,pid,tid,time,period,event,ip,sym,dso,trace > $outputDumpFile 2>>$logFile
            LogAppend
        fi

        # If the dump file is zero length, try to collect without the period field, which was added recently.
        if [ ! -s $outputDumpFile ]
        then
            LogAppend "Running $perfcmd script -i $inputFile --fields comm,pid,tid,time,event,ip,sym,dso,trace > $outputDumpFile"
            $perfcmd script -i $inputFile --fields comm,pid,tid,time,event,ip,sym,dso,trace > $outputDumpFile 2>>$logFile
            LogAppend
        fi

        WriteStatus "...FINISHED"
    fi

    WriteStatus "Compressing trace files"
    
    # Move all collected files to the new directory.
    RunSilent "mv -f * $directoryName"

    # Close the log - this stops all writing to the log, so that we can move it into the archive.
    CloseLog

    # Move the log file to the new directory and rename it to the standard log name.
    RunSilent "mv $logFile $directoryName/perfcollect.log"

    # Compress the data.
    local archiveSuffix=".zip"
    local archiveName=$directoryName$archiveSuffix
    RunSilent "$zipcmd -r $archiveName $directoryName"

    # Move back to the original directory.
    popd > /dev/null

    # Move the archive.
    RunSilent "mv $tempDir/$archiveName ."

    WriteStatus "...FINISHED"

    WriteStatus "Cleaning up artifacts"

    # Delete the temp directory.
    RunSilent "rm -rf $tempDir $collectInfoFile"

    WriteStatus "...FINISHED"

    # Tell the user where the trace is.
    WriteStatus
    WriteStatus "Trace saved to $archiveName"
}

##
# Handle the CTRL+C signal.
##
CTRLC_Handler()
{
    # Mark the handler invoked.
    handlerInvoked=1
}

EndCollect()
{
    # The user must either use "collect stop" or CTRL+C to stop collection.
    if [ "$action" == "stop" ]
    then 
        # Recover trace info in the previous program.
        source $collectInfoFile  # TODO: exit and dispose upon missing file
        # New program started, so go back to the temp directory.
        pushd $tempDir > /dev/null
    fi

    if [ "$useLTTng" == "1" ]
    then
        StopLTTngCollection
    fi

    # Update the user.
    WriteStatus
    WriteStatus "...STOPPED."
    WriteStatus
    WriteStatus "Starting post-processing.  This may take some time."
    WriteStatus

    # The user used CTRL+C to stop collection.
    # When this happens, we catch the signal and finish our work.
    ProcessCollectedData
}

##
# Print usage information.
##
PrintUsage()
{
    echo "This script uses perf_event and LTTng to collect and view performance traces for .NET applications."
    echo "For detailed collection and viewing steps, view this script in a text editor or viewer."
    echo ""
    echo "./perfcollect <action> <tracename>"
    echo "Valid Actions: collect start/stop view livetrace install"
    echo ""
    echo "collect options:"
    echo "By default, collection includes CPU samples collected every ms."
    echo "    -pid          : Only collect data from the specified process id."
    echo "    -threadtime   : Collect events for thread time analysis (on and off cpu)."
    echo "    -offcpu       : Collect events for off-cpu analysis."
    echo "    -hwevents     : Collect (some) hardware counters."
    echo "    -buffersize   : Set the size of the buffer used by perf_events.  Default size is specified in pages."
    echo "                    Specify B/K/M/G suffix for bytes/kilobytes/megabytes/gigabytes."
    echo "                    Example: -buffersize 100M for 100 megabytes"
    echo ""
    echo "start:"
    echo "    Start collection, but with Lttng trace ONLY. It needs to be used with 'stop' action."
    echo ""
    echo "stop:"
    echo "    Stop collection if 'start' action is used."
    echo ""
    echo "view options:"
    echo "    -processfilter      : Filter data by the specified process name."
    echo "    -graphtype      : Specify the type of graph.  Valid values are 'caller' and 'callee'.  Default is 'callee'."
    echo "    -viewer          : Specify the data viewer.  Valid values are 'perf' and 'lttng'.  Default is 'perf'."
    echo ""
    echo "livetrace:"
    echo "    Print EventSource events directly to the console.  Root privileges not required."
    echo ""
    echo "install options:"
    echo "    Useful for first-time setup.  Installs/upgrades perf_event and LTTng."
    echo "    -force    : Force installation of required packages without prompt"
    echo ""
}

##
# Validate and set arguments.
##

BuildPerfRecordArgs()
{
    # Start with default collection arguments that record at realtime priority, all CPUs (-a), and collect call stacks (-g)
    collectionArgs="record $bufferSize -k 1 -g"

    # Filter to a single process if desired
    if [ "$collectionPid" != "" ]
    then
        if ps -p "$collectionPid" > /dev/null;
        then
            collectionArgs="$collectionArgs --pid=$collectionPid"
        else
            WriteWarning "Process with PID '$collectionPid' does not exist.  Please specify a valid PID."
            exit 1
        fi
    else
        collectionArgs="$collectionArgs -a"
    fi

    # Enable CPU Collection
    if [ $collect_cpu -eq 1 ]
    then
        collectionArgs="$collectionArgs"
        eventsToCollect=( "${eventsToCollect[@]}" "cpu-clock" )

        # If only collecting CPU events, set the sampling rate to 1000.
        # Otherwise, use the default sampling rate to avoid sampling sched events.
        if [ $collect_threadTime -eq 0 ] && [ $collect_offcpu -eq 0 ]
        then
            collectionArgs="$collectionArgs -F 1000"
        fi
    fi

    if [ $collect_system -eq 1 ]
    then
        collectionArgs="$collectionArgs -e major-faults -e minor-faults"
    fi

    # Enable HW counters event collection
    if [ $collect_HWevents -eq 1 ]
    then
        collectionArgs="$collectionArgs -e cycles,instructions,branches,cache-misses"
    fi
    
    # Enable context switches.
    if [ $collect_threadTime -eq 1 ]
    then
        eventsToCollect=( "${eventsToCollect[@]}" "sched:sched_stat_sleep" "sched:sched_switch" "sched:sched_process_exit" )
    fi

    # Enable offcpu collection
    if [ $collect_offcpu -eq 1 ]
    then
        eventsToCollect=( "${eventsToCollect[@]}" "sched:sched_stat_sleep" "sched:sched_switch" "sched:sched_process_exit" )
    fi

    # Build up the set of events.
    local eventString=""
    local comma=","
    for (( i=0; i<${#eventsToCollect[@]}; i++ ))
    do
        # Get the arg.
        eventName=${eventsToCollect[$i]}

        # Build up the comma separated list.
        if [ "$eventString" == "" ]
        then
            eventString=$eventName
        else
            eventString="$eventString$comma$eventName"
        fi

    done

    if [ ! -z ${duration} ]
    then
        durationString="sleep ${duration}"
    fi

    # Add the events onto the collection command line args.    
    collectionArgs="$collectionArgs -e $eventString $durationString"
}

DoCollect()
{
    # Ensure the script is run as root.
    EnsureRoot

    # Build collection args.
    # Places the resulting args in $collectionArgs
    BuildPerfRecordArgs

    # Trap CTRL+C
    trap CTRLC_Handler SIGINT

    # Create a temp directory to use for collection.
    local tempDir=`mktemp -d`
    LogAppend "Created temp directory $tempDir"

    # Switch to the directory.
    pushd $tempDir > /dev/null

    # Start LTTng collection.
    if [ "$useLTTng" == "1" ]
    then
        StartLTTngCollection
    fi

    # Tell the user that collection has started and how to exit.
    if [ "$duration" != "" ]
    then
        WriteStatus "Collection started. Collection will automatically stop in $duration second(s).  Press CTRL+C to stop early."
    elif [ "$action" == "start" ]
    then
        WriteStatus "Collection started."
    else
        WriteStatus "Collection started. Press CTRL+C to stop."
    fi

    # Start perf record.
    if [ "$action" == "start" ]
    then 
        # Skip perf, which can only be stopped by CTRL+C.
        # Pass trace directory and session name to the stop process.
        popd > /dev/null
        usePerf=0
        declare -p | grep -e lttngTraceDir -e lttngSessionName -e tempDir -e usePerf -e useLttng -e logFile > $collectInfoFile
    else
        if [ "$usePerf" == "1" ]
        then
            if [ ! -z ${duration} ]
            then
                RunSilent $perfcmd $collectionArgs
            else
                RunSilentBackground $perfcmd $collectionArgs
            fi
        else
            # Wait here until CTRL+C handler gets called when user types CTRL+C.
            LogAppend "Waiting for CTRL+C handler to get called."

            waitTime=0
            for (( ; ; ))
            do
                if [ "$handlerInvoked" == "1" ]
                then
                    break;
                fi

                # Wait and then check to see if the handler has been invoked or we've crossed the duration threshold.
                sleep 1
                waitTime=$waitTime+1
                if (( duration > 0 && duration <= waitTime ))
                then
                    break;
                fi
            done
        fi
        # End collection if action is 'collect'.
        EndCollect
    fi
}

DoLiveTrace()
{
    # Start the session
    StartLTTngCollection

    # View the event stream (until the user hits CTRL+C)
    WriteStatus "Listening for events from LTTng.  Hit CTRL+C to stop."
    $lttngcmd view

    # Stop the LTTng sessoin
    StopLTTngCollection
}

# $1 == Path to directory containing trace files
PropSymbolsAndMapFilesForView()
{
    # Get the current directory
    local currentDir=`pwd`    

    # Copy map files to /tmp since they aren't supported by perf buildid-cache.
    local mapFiles=`find -name *.map`
    for mapFile in $mapFiles
    do
        echo "Copying $mapFile to /tmp."
        cp $mapFile /tmp
    done

    # Cache all debuginfo files saved with the trace in the buildid cache.
    local debugInfoFiles=`find $currentDir -name *.debuginfo`
    for debugInfoFile in $debugInfoFiles
    do
        echo "Caching $debugInfoFile in buildid cache using perf buildid-cache."
        $perfcmd buildid-cache --add=$debugInfoFile
    done
}

SetEnvironment()
{
    WriteStatus "Starting a new shell configured for profiling."
    WriteStatus "When done, type 'exit' to exit the profiling shell."
    bash -c 'export COMPlus_PerfMapEnabled=1;export COMPlus_EnableEventLog=1; exec bash'
    WriteStatus "Profiling shell exited."
}

DoView()
{
    # Generate a temp directory to extract the trace files into.
    local tempDir=`mktemp -d`
    
    # Extract the trace files.
    $unzipcmd $inputTraceName -d $tempDir
    
    # Move the to temp directory.
    pushd $tempDir
    cd `ls`

    # Select the viewer.
    if [ "$viewer" == "perf" ]
    then
        # Prop symbols and map files.
        PropSymbolsAndMapFilesForView `pwd`

        # Choose the view
        if [ "$graphType" == "" ]
        then
            graphType="callee"
        elif [ "$graphType" != "callee" ] && [ "$graphType" != "caller"]
        then
            FatalError "Invalid graph type specified.  Valid values are 'callee' and 'caller'."
        fi

        # Filter to specific process names if desired.
        if [ "$processFilter" != "" ]
        then
            processFilter="--comms=$processFilter"
        fi

        # Execute the viewer.
        $perfcmd report -n -g graph,0.5,$graphType $processFilter $perfOpt
    elif [ "$viewer" == "lttng" ]
    then
        babeltrace lttngTrace/ | more
    fi
    
    # Switch back to the original directory.
    popd

    # Delete the temp directory.
    rm -rf $tempDir
}

#####################################
## Main Script Start
#####################################

# No arguments
if [ "$#" == "0" ]
then
    PrintUsage
    exit 0
fi

# Install perf if requested.  Do this before all other validation.
if [ "$1" == "install" ]
then
    if [ "$2" == "-force" ]
    then 
        forceInstall=1
    fi
    InstallPerf
    InstallLTTng
    exit 0
fi

# Discover external commands that will be called by this script.
DiscoverCommands

# Initialize the log.
if [ "$1" != "stop" ]
then
    InitializeLog
fi

# Process arguments.
ProcessArguments $@

# Ensure prerequisites are installed.
EnsurePrereqsInstalled

# Take the appropriate action.
if [ "$action" == "collect" ] || [ "$action" == "start" ]
then
    DoCollect
elif [ "$action" == "stop" ]
then
    EndCollect
elif [ "$action" == "setenv" ]
then
    SetEnvironment
elif [ "$action" == "view" ]
then
    DoView
elif [ "$action" == "livetrace" ]
then
    DoLiveTrace
fi