Enable EventPipe across Unix and Windows (#14772)

* [squashed] most work complete to enable EventPipe on Windows

* Eventpipe now builds on Windows

* Ensure evevntpipe is intialized on Windows

* Resolve the location of python from build.cmd

* Ensure eventing files are generated when needed

* moving linkage declaration to cmake

* create new event from constructor

* enable FEATURE_EVENT_TRACE and FEATURE_PERF_TRACE everywhere

* [WIP] checkpoint in fixing contarct errors

* add another possible python location

* Fix double delete bug in EventPipeConfiguration destructor

* Fix typo in function name

* Revert changes to .gitgnore

* bump to netstandard1.6 in preperation for new version of TraceEvent

* Revert changes to groovy files

* revert changes to perf-prep scripts

* add common.h and use nothrow

* Fix issue which was causing double delete of configprovider

* Add new test utilizing TraceEvent

* Remove accidentally added local directory reference

* Add comment to explain the addition of misc/tracepointprovider.cpp

* Add back sys.exit(0) and refactor

* Change conditional to be more direct

* Better handle NULL config

* Initialize m_deleteDefered

* Eliminate obsolete field

* Fix spelling error

* Fix nits

* Make smaple progiler timing functions easier to read

* Move projects back to netstandard1.4

* Incomplete improvements to EventPipeTrace test

* Add event integrity checks to test

* Clean up some left over code

* Add EventSource based test

* Remove unused PAL tests on Windows

* Fix Linux build breaks

* Minor changes to CMake files

* Remove //HACK for hack that was previously removed

* Fix formatting and negate a #ifdef

* Add conditional to ensure PERFTRACING is not enabled without EVENT_TRACE

* Take lock on EventPipeProvider and EventPipeConfiguration destruction

* Load winmm.dll at runtime

* Change function name and compile conditions

* Move typedef into #ifndef

* Use the correct config in setup

* Change lifecycle managment of EventPipeConfiguration's configuration provider

* Enable EventPipe tests pri0 and disable broken tests

* Replace python3 only error with python2 compatable one

* Make common.csproj build pri0

* Change TraceEvent version to 2.0.2 to match published verison

* Address cross build failure

* Remove use of undefined variable

* Add crossgen fix to .cmd

* Use more specific types to avoid marshalling errors

* Use Assert-style statements and remove one check

* Fix cross arch build

* Fix flipped branch

* Bring build.cmd changes to build.sh

* Fix cmake writing

* Revert "Bring build.cmd changes to build.sh"

This reverts commit 893c649.

* remove stdlib.h

* Fix out of order null check

Author: Victor "Nate" Graf

CMakeLists.txt

@@ -23,6 +23,7 @@ endif (WIN32)
 set(CLR_DIR ${CMAKE_CURRENT_SOURCE_DIR})
 set(VM_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src/vm)
 set(GENERATED_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/src/inc)
+set(GENERATED_EVENTING_DIR ${CMAKE_CURRENT_BINARY_DIR}/eventing)
 set(VERSION_FILE_PATH "${CMAKE_BINARY_DIR}/version.cpp")
 
 set(CORECLR_SET_RPATH ON)

build.cmd

@@ -28,7 +28,7 @@ set __ThisScriptDir="%~dp0"
 if defined VisualStudioVersion (
     if not defined __VSVersion echo %__MsgPrefix%Detected Visual Studio %VisualStudioVersion% developer command ^prompt environment
     goto :Run
-) 
+)
 
 echo %__MsgPrefix%Searching ^for Visual Studio 2017 or 2015 installation
 set _VSWHERE="%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe"
@@ -294,6 +294,16 @@ if %__EnforcePgo%==1 (
     )
 )
 
+REM Determine if this is a cross-arch build
+
+if /i "%__BuildArch%"=="arm64" (
+    set __DoCrossArchBuild=1
+    )
+
+if /i "%__BuildArch%"=="arm" (
+    set __DoCrossArchBuild=1
+    )
+
 :: Set the remaining variables based upon the determined build configuration
 set "__BinDir=%__RootBinDir%\Product\%__BuildOS%.%__BuildArch%.%__BuildType%"
 set "__IntermediatesDir=%__RootBinDir%\obj\%__BuildOS%.%__BuildArch%.%__BuildType%"
@@ -380,6 +390,38 @@ for /f "tokens=*" %%s in ('%DotNetCli% msbuild "%OptDataProjectFilePath%" /t:Dum
     set __IbcOptDataVersion=%%s
 )
 
+REM =========================================================================================
+REM ===
+REM === Generate source files for eventing
+REM ===
+REM =========================================================================================
+
+REM Find python and set it to the variable PYTHON
+echo import sys; print sys.executable | (python2.7 || python2 || py -2 || python) > %TEMP%\pythonlocation.txt 2> NUL
+set /p PYTHON=<%TEMP%\pythonlocation.txt
+if NOT DEFINED PYTHON (
+    echo %__MsgPrefix%Error: Could not find a python 2.7 installation
+    exit /b 1
+)
+
+if /i "%__DoCrossArchBuild%"=="1" (
+    set __IntermediatesIncDir=%__CrossCompIntermediatesDir%\src\inc
+    set __IntermediatesEventingDir=%__CrossCompIntermediatesDir%\eventing
+) else (
+    set __IntermediatesIncDir=%__IntermediatesDir%\src\inc
+    set __IntermediatesEventingDir=%__IntermediatesDir%\eventing
+)
+
+echo Laying out dynamically generated files consumed by the build system
+echo Laying out dynamically generated Event test files and etmdummy stub functions
+%PYTHON% -B -Wall  %__SourceDir%\scripts\genEventing.py --inc %__IntermediatesIncDir% --dummy %__IntermediatesIncDir%\etmdummy.h --man %__SourceDir%\vm\ClrEtwAll.man --nonextern || exit /b 1
+
+echo Laying out dynamically generated EventPipe Implementation
+%PYTHON% -B -Wall %__SourceDir%\scripts\genEventPipe.py --man %__SourceDir%\vm\ClrEtwAll.man --intermediate %__IntermediatesEventingDir%\eventpipe --nonextern || exit /b 1
+
+echo Laying out ETW event logging interface
+%PYTHON% -B -Wall %__SourceDir%\scripts\genEtwProvider.py --man %__SourceDir%\vm\ClrEtwAll.man --intermediate %__IntermediatesIncDir% --exc %__SourceDir%\vm\ClrEtwAllMeta.lst || exit /b 1
+
 REM =========================================================================================
 REM ===
 REM === Build the CLR VM
@@ -476,14 +518,6 @@ REM === Build Cross-Architecture Native Components (if applicable)
 REM ===
 REM =========================================================================================
 
-if /i "%__BuildArch%"=="arm64" (
-    set __DoCrossArchBuild=1
-    )
-
-if /i "%__BuildArch%"=="arm" (
-    set __DoCrossArchBuild=1
-    )
-
 if /i "%__DoCrossArchBuild%"=="1" (
     REM Scope environment changes start {
     setlocal

build.sh

@@ -194,28 +194,9 @@ generate_event_logging_sources()
     fi
 
 # Event Logging Infrastructure
-   __GeneratedIntermediate="$__IntermediatesDir/Generated"
-   __GeneratedIntermediateEventProvider="$__GeneratedIntermediate/eventprovider_new"
-   __GeneratedIntermediateEventPipe="$__GeneratedIntermediate/eventpipe_new"
-
-    if [[ -d "$__GeneratedIntermediateEventProvider" ]]; then
-        rm -rf  "$__GeneratedIntermediateEventProvider"
-    fi
-
-    if [[ -d "$__GeneratedIntermediateEventPipe" ]]; then
-        rm -rf  "$__GeneratedIntermediateEventPipe"
-    fi
-
-    if [[ ! -d "$__GeneratedIntermediate/eventprovider" ]]; then
-        mkdir -p "$__GeneratedIntermediate/eventprovider"
-    fi
-
-    if [[ ! -d "$__GeneratedIntermediate/eventpipe" ]]; then
-        mkdir -p "$__GeneratedIntermediate/eventpipe"
-    fi
-
-    mkdir -p "$__GeneratedIntermediateEventProvider"
-    mkdir -p "$__GeneratedIntermediateEventPipe"
+    __GeneratedIntermediate="$__IntermediatesDir/eventing"
+    __GeneratedIntermediateEventProvider="$__GeneratedIntermediate/eventprovider"
+    __GeneratedIntermediateEventPipe="$__GeneratedIntermediate/eventpipe"
 
     __PythonWarningFlags="-Wall"
     if [[ $__IgnoreWarnings == 0 ]]; then
@@ -225,54 +206,38 @@ generate_event_logging_sources()
 
     if [[ $__SkipCoreCLR == 0 || $__ConfigureOnly == 1 ]]; then
         echo "Laying out dynamically generated files consumed by the build system "
-        echo "Laying out dynamically generated Event Logging Test files"
-        $PYTHON -B $__PythonWarningFlags "$__ProjectRoot/src/scripts/genXplatEventing.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --exc "$__ProjectRoot/src/vm/ClrEtwAllMeta.lst" --testdir "$__GeneratedIntermediateEventProvider/tests"
+        echo "Laying out dynamically generated Event test files, etmdummy stub functions, and external linkages"
+        $PYTHON -B $__PythonWarningFlags "$__ProjectRoot/src/scripts/genEventing.py" --inc $__IntermediatesDir/src/inc --dummy $__IntermediatesDir/src/inc/etmdummy.h --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --testdir "$__GeneratedIntermediateEventProvider/tests"
 
         if  [[ $? != 0 ]]; then
             exit
         fi
 
-        case $__BuildOS in
-            Linux|FreeBSD)
-                echo "Laying out dynamically generated EventPipe Implementation"
-                $PYTHON -B $__PythonWarningFlags "$__ProjectRoot/src/scripts/genEventPipe.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --intermediate "$__GeneratedIntermediateEventPipe" --exc "$__ProjectRoot/src/vm/ClrEtwAllMeta.lst"
-                if  [[ $? != 0 ]]; then
-                    exit
-                fi
-                ;;
-            *)
-                ;;
-        esac
+        echo "Laying out dynamically generated EventPipe Implementation"
+        $PYTHON -B $__PythonWarningFlags "$__ProjectRoot/src/scripts/genEventPipe.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --intermediate "$__GeneratedIntermediateEventPipe"
 
         #determine the logging system
         case $__BuildOS in
             Linux|FreeBSD)
                 echo "Laying out dynamically generated Event Logging Implementation of Lttng"
-                $PYTHON -B $__PythonWarningFlags "$__ProjectRoot/src/scripts/genXplatLttng.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --intermediate "$__GeneratedIntermediateEventProvider"
+                $PYTHON -B $__PythonWarningFlags "$__ProjectRoot/src/scripts/genLttngProvider.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --intermediate "$__GeneratedIntermediateEventProvider"
                 if  [[ $? != 0 ]]; then
                     exit
                 fi
                 ;;
             *)
+                echo "Laying out dummy event logging provider"
+                $PYTHON -B $__PythonWarningFlags "$__ProjectRoot/src/scripts/genDummyProvider.py" --man "$__ProjectRoot/src/vm/ClrEtwAll.man" --intermediate "$__GeneratedIntermediateEventProvider"
+                if  [[ $? != 0 ]]; then
+                    exit
+                fi
                 ;;
         esac
-    fi
-
-    echo "Cleaning the temp folder of dynamically generated Event Logging files"
-    $PYTHON -B $__PythonWarningFlags -c "import sys;sys.path.insert(0,\"$__ProjectRoot/src/scripts\"); from Utilities import *;UpdateDirectory(\"$__GeneratedIntermediate/eventprovider\",\"$__GeneratedIntermediateEventProvider\")"
-    if  [[ $? != 0 ]]; then
-        exit
-    fi
 
-    rm -rf "$__GeneratedIntermediateEventProvider"
-
-    echo "Cleaning the temp folder of dynamically generated EventPipe files"
-    $PYTHON -B $__PythonWarningFlags -c "import sys;sys.path.insert(0,\"$__ProjectRoot/src/scripts\"); from Utilities import *;UpdateDirectory(\"$__GeneratedIntermediate/eventpipe\",\"$__GeneratedIntermediateEventPipe\")"
-    if  [[ $? != 0 ]]; then
-        exit
+        if [[ $__CrossBuild == 1 ]]; then
+            cp -r $__GeneratedIntermediate $__CrossCompIntermediatesDir
+        fi
     fi
-
-    rm -rf "$__GeneratedIntermediateEventPipe"
 }
 
 build_native()

clr.coreclr.props

@@ -3,6 +3,8 @@
     <FeatureEventTrace>true</FeatureEventTrace>
     <FeatureICastable>true</FeatureICastable>
     <FeatureManagedEtwChannels>true</FeatureManagedEtwChannels>
+    <FeatureManagedEtw>true</FeatureManagedEtw>
+    <FeaturePerfTracing>true</FeaturePerfTracing>
 
     <ProfilingSupportedBuild>true</ProfilingSupportedBuild>
   </PropertyGroup>
@@ -21,7 +23,6 @@
   <PropertyGroup Condition="'$(TargetsWindows)' == 'true'">
     <FeatureArrayStubAsIL Condition="'$(TargetArch)' != 'i386'">true</FeatureArrayStubAsIL>
     <FeatureMulticastStubAsIL Condition="'$(TargetArch)' != 'i386'">true</FeatureMulticastStubAsIL>
-    <FeatureManagedEtw>true</FeatureManagedEtw>
     <FeatureStubsAsIL Condition="'$(TargetArch)' == 'arm64'">true</FeatureStubsAsIL>
     <FeatureUseLcid>true</FeatureUseLcid>
     <FeatureCominterop>true</FeatureCominterop>
@@ -32,9 +33,4 @@
     <FeatureAppX>true</FeatureAppX>
     <FeatureWin32Registry>true</FeatureWin32Registry>
   </PropertyGroup>
-
-  <PropertyGroup Condition="'$(TargetsLinux)' == 'true'">
-    <FeatureManagedEtw>true</FeatureManagedEtw>
-    <FeaturePerfTracing>true</FeaturePerfTracing>
-  </PropertyGroup>
-</Project>
+</Project>

clr.defines.targets

@@ -20,6 +20,7 @@
         <DefineConstants Condition="'$(FeatureManagedEtwChannels)' == 'true'">$(DefineConstants);FEATURE_MANAGED_ETW_CHANNELS</DefineConstants>
         <DefineConstants Condition="'$(FeaturePal)' == 'true'">$(DefineConstants);FEATURE_PAL</DefineConstants>
         <DefineConstants Condition="'$(FeaturePerfTracing)' == 'true'">$(DefineConstants);FEATURE_PERFTRACING</DefineConstants>
+        <DefineConstants Condition="'$(FeatureEventTrace)' == 'true'">$(DefineConstants);FEATURE_EVENT_TRACE</DefineConstants>
         <DefineConstants Condition="'$(FeatureXplatEventSource)' == 'true'">$(DefineConstants);FEATURE_EVENTSOURCE_XPLAT</DefineConstants>
         <DefineConstants Condition="'$(FeatureUseLcid)' == 'true'">$(DefineConstants);FEATURE_USE_LCID</DefineConstants>
         <DefineConstants Condition="'$(FeatureWin32Registry)' == 'true'">$(DefineConstants);FEATURE_WIN32_REGISTRY</DefineConstants>

clrdefinitions.cmake

@@ -125,6 +125,7 @@ if(FEATURE_DBGIPC)
 endif(FEATURE_DBGIPC)
 if(FEATURE_EVENT_TRACE)
     add_definitions(-DFEATURE_EVENT_TRACE=1)
+    add_definitions(-DFEATURE_PERFTRACING=1)
 endif(FEATURE_EVENT_TRACE)
 if(FEATURE_GDBJIT)
     add_definitions(-DFEATURE_GDBJIT)
@@ -138,9 +139,6 @@ endif(FEATURE_GDBJIT_LANGID_CS)
 if(FEATURE_GDBJIT_SYMTAB)
     add_definitions(-DFEATURE_GDBJIT_SYMTAB)
 endif(FEATURE_GDBJIT_SYMTAB)
-if(CLR_CMAKE_PLATFORM_LINUX)
-    add_definitions(-DFEATURE_PERFTRACING)
-endif(CLR_CMAKE_PLATFORM_LINUX)
 if(CLR_CMAKE_PLATFORM_UNIX)
     add_definitions(-DFEATURE_EVENTSOURCE_XPLAT=1)
 endif(CLR_CMAKE_PLATFORM_UNIX)

clrfeatures.cmake

@@ -3,23 +3,13 @@ if(CLR_CMAKE_TARGET_TIZEN_LINUX)
 endif()
 
 if(NOT DEFINED FEATURE_EVENT_TRACE)
-  if (WIN32)
-    set(FEATURE_EVENT_TRACE 1)
-  endif()
-
-  if(CLR_CMAKE_PLATFORM_LINUX)
-    if(CLR_CMAKE_TARGET_TIZEN_LINUX)
-      set(FEATURE_EVENT_TRACE 1)
-    elseif(CLR_CMAKE_TARGET_ARCH_AMD64)
-      set(FEATURE_EVENT_TRACE 1)
-    elseif(CLR_CMAKE_TARGET_ARCH_ARM)
-      set(FEATURE_EVENT_TRACE 1)
-    elseif(CLR_CMAKE_TARGET_ARCH_ARM64)
-      set(FEATURE_EVENT_TRACE 1)
-    endif()
-  endif(CLR_CMAKE_PLATFORM_LINUX)
+  set(FEATURE_EVENT_TRACE 1)
 endif(NOT DEFINED FEATURE_EVENT_TRACE)
 
+if(NOT DEFINED FEATURE_PERFTRACING AND FEATURE_EVENT_TRACE)
+  set(FEATURE_PERFTRACING 1)
+endif(NOT DEFINED FEATURE_PERFTRACING AND FEATURE_EVENT_TRACE)
+
 if(NOT DEFINED FEATURE_DBGIPC)
   if(CLR_CMAKE_PLATFORM_UNIX AND (NOT CLR_CMAKE_PLATFORM_ANDROID))
     set(FEATURE_DBGIPC 1)

dependencies.props

@@ -38,7 +38,7 @@
     <XunitPackageVersion>2.2.0-beta2-build3300</XunitPackageVersion>
     <XunitConsoleNetcorePackageVersion>1.0.2-prerelease-00177</XunitConsoleNetcorePackageVersion>
     <XunitPerformanceApiPackageVersion>1.0.0-beta-build0012</XunitPerformanceApiPackageVersion>
-    <MicrosoftDiagnosticsTracingTraceEventPackageVersion>1.0.3-alpha-experimental</MicrosoftDiagnosticsTracingTraceEventPackageVersion>
+    <MicrosoftDiagnosticsTracingTraceEventPackageVersion>2.0.2</MicrosoftDiagnosticsTracingTraceEventPackageVersion>
     <VCRuntimeVersion>1.2.0</VCRuntimeVersion>
   </PropertyGroup>