Make the sgx debug flag dependent on PDO_DEBUG_BUILD.

If PDO_DEBUG_BUILD is not set or set to 0, the enclave is built with
SGX_DEBUG_FLAG set to 0, and signed with the DisableDebug flag set to 1.
So this commit adds one more step in the enclave cmake build to
create the xml configuration file accordingly.

Co-authored-by: Mic Bowman <mic.bowman@intel.com>
Signed-off-by: Bruno Vavala <bruno.vavala@intel.com>

.github/workflows/ci.yaml

@@ -23,6 +23,8 @@ jobs:
               if: "!contains(github.event.commits[0].message, '[debug]')"
               env:
                   PDO_INTERPRETER: ${{ matrix.interpreter }}
+                  PDO_DEBUG_BUILD: 1
+                  PDO_LOG_LEVEL: warning
               run: |
                   # The creation of a dummy branch is necessary for the CI tests
                   # to work on PRs. Based on empirical results, in the absence of

build/__tools__/build.sh

@@ -52,6 +52,25 @@ else
     MAKE_ARGS="-j${NUM_CORES} BUILD_CLIENT=0"
 fi
 
+# Set CMAKE built type through the CMAKE_BUILD_TYPE variable; this works
+# for cmake version 3.22; later versions can use --config <Debug|Release>
+#
+# Note that the build type affects the SGX debug flag:
+#  * Build type "Release" defines NDEBUG, which causes sgx_urts.h to set SGX_DEBUG_FLAG to 0.
+#  * Build type "Debug" does not define NDEBUG, which causes sgx_urts.h to set SGX_DEBUG_FLAG to 1.
+#
+# We do not need to explicitly add these to the make args; as environment variables they will be
+# passed through appropriately
+if [ ${PDO_DEBUG_BUILD} == "0" ]; then
+    CMAKE_ARGS+=" -DCMAKE_BUILD_TYPE=Release"
+else
+    CMAKE_ARGS+=" -DCMAKE_BUILD_TYPE=Debug"
+fi
+
+# We need to export CMAKE_ARGS so that the Makefiles that call
+# cmake (e.g. pservice and eservice) will pick up these definitions
+export CMAKE_ARGS
+
 # -----------------------------------------------------------------
 # BUILD
 # -----------------------------------------------------------------

build/cmake/ProjectVariables.cmake

@@ -20,7 +20,7 @@
 ADD_COMPILE_OPTIONS(-m64 -fvisibility=hidden -fpie -fPIC -fstack-protector)
 ADD_COMPILE_OPTIONS($<$<COMPILE_LANGUAGE:CXX>:-std=c++11>)
 
-OPTION(PDO_DEBUG_BUILD "Build with debugging turned on" FALSE)
+OPTION(PDO_DEBUG_BUILD "Build with debugging turned on" 0)
 
 IF (DEFINED ENV{PDO_DEBUG_BUILD})
   SET(PDO_DEBUG_BUILD $ENV{PDO_DEBUG_BUILD})

build/cmake/SGX.cmake

@@ -26,9 +26,24 @@ IF (NOT DEFINED ENV{SGX_MODE})
 ENDIF()
 SET(SGX_MODE $ENV{SGX_MODE})
 
+# There are effectively three build modes for SGX:
+#   1) SIM mode with PDO_DEBUG_BUILD enabled
+#   2) HW mode with PDO_DEBUG_BUILD enabled
+#   3) HW mode with PDO_DEBUG_BUILD disabled (release mode)
+# For now we just check the consistency of the variables (SGX_MODE, PDO_DEBUG_BUILD and CMAKE_BUIDL_TYPE)
 IF (${SGX_MODE} STREQUAL "SIM")
+    IF (NOT ${PDO_DEBUG_BUILD})
+        MESSAGE(FATAL_ERROR "SGX_MODE=SIM does not accept PDO_DEBUG_BUILD=0")
+    ENDIF()
+
     SET(SGX_USE_SIMULATOR TRUE)
 ELSE()
+    IF (${CMAKE_BUILD_TYPE} STREQUAL "Release")
+        IF (${PDO_DEBUG_BUILD})
+            MESSAGE(FATAL_ERROR "SGX_MODE=HW and CMAKE_BUILD_TYPE=Release do not accept PDO_DEBUG_BUILD=1")
+        ENDIF()
+    ENDIF()
+
     SET(SGX_USE_SIMULATOR FALSE)
 ENDIF()
 
@@ -241,3 +256,34 @@ FUNCTION(SGX_DEPLOY_FILES TARGET HEADER_NAME)
     COMMAND ${GENERATE_COMMAND} --metadata ${SIGNED_ENCLAVE}.meta --header ${HEADER_FILE} --enclave ${VARIABLE_NAME}
   )
 ENDFUNCTION()
+
+# -----------------------------------------------------------------
+# SGX_CONFIGURE_ENCLAVE
+#
+# Generate the xml configuration file which can be then used by
+# SGX_SIGN. For now, the only field that is changed is the flag to
+# disable debugging. Note that this uses the configure_file cmake
+# command which expands references to variables like ${DISABLE_DEBUG}
+# -----------------------------------------------------------------
+FUNCTION(SGX_CONFIGURE_ENCLAVE TARGET CONFIG)
+  # the flag in the configuration file is used to DISABLE debugging,
+  # precisely the opposite of the PDO_DEBUG_BUILD flag
+  IF (${PDO_DEBUG_BUILD})
+    SET(DISABLE_DEBUG "0")
+  ELSE()
+    SET(DISABLE_DEBUG "1")
+  ENDIF()
+
+  CONFIGURE_FILE(${CONFIG}.in ${CONFIG})
+
+  ADD_CUSTOM_TARGET(
+    generate-configuration-file
+    DEPENDS ${CONFIG})
+
+  SET_PROPERTY(
+    TARGET generate-configuration-file
+    APPEND
+    PROPERTY ADDITIONAL_CLEAN_FILES ${CONFIG})
+
+  ADD_DEPENDENCIES(${TARGET} generate-configuration-file)
+ENDFUNCTION()

docker/Makefile

@@ -24,6 +24,10 @@
 # the desired PDO_SOURCE_ROOT when invoking make.
 PDO_SOURCE_ROOT ?= $(realpath $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/../)
 
+# PDO_DEBUG_BUILD
+# The flag that determines the build type (debug or not).
+PDO_DEBUG_BUILD ?= ${PDO_DEBUG_BUILD:-0}
+
 # PDO_REPO --
 #   The URL or path to the git repository, the default uses the current repository
 #   though it might be reasonable to build a particular branch from the primary
@@ -69,12 +73,14 @@ rebuild_% : repository
 	$(DOCKER_COMMAND) build $(DOCKER_ARGS) \
 		--build-arg REBUILD=$(TIMESTAMP) \
 		--build-arg PDO_VERSION=$(PDO_VERSION) \
+		--build-arg PDO_DEBUG_BUILD=$(PDO_DEBUG_BUILD) \
 		--tag pdo_$*:$(PDO_VERSION) \
 		--file '$(DOCKER_DIR)'/pdo_$*.dockerfile .
 
 build_% : repository
 	$(DOCKER_COMMAND) build $(DOCKER_ARGS) \
 		--build-arg PDO_VERSION=$(PDO_VERSION) \
+		--build-arg PDO_DEBUG_BUILD=$(PDO_DEBUG_BUILD) \
 		--tag pdo_$*:$(PDO_VERSION) \
 		--file '$(DOCKER_DIR)'/pdo_$*.dockerfile .
 
@@ -83,13 +89,15 @@ rebuild_services_sgx : repository
 	$(DOCKER_COMMAND) build $(DOCKER_ARGS) \
 		--build-arg REBUILD=$(TIMESTAMP) \
 		--build-arg PDO_VERSION=$(PDO_VERSION) \
+		--build-arg PDO_DEBUG_BUILD=$(PDO_DEBUG_BUILD) \
 		--build-arg SGX_MODE=HW \
 		--tag pdo_services_sgx:$(PDO_VERSION) \
 		--file $(DOCKER_DIR)/pdo_services.dockerfile .
 
 build_services_sgx : $(IAS_CERTIFICATES) repository build_services_base
 	$(DOCKER_COMMAND) build $(DOCKER_ARGS) \
 		--build-arg PDO_VERSION=$(PDO_VERSION) \
+		--build-arg PDO_DEBUG_BUILD=$(PDO_DEBUG_BUILD) \
 		--build-arg SGX_MODE=HW \
 		--tag pdo_services_sgx:$(PDO_VERSION) \
 		--file $(DOCKER_DIR)/pdo_services.dockerfile .

eservice/.gitignore

@@ -6,3 +6,4 @@ dist
 *.pyc
 *.so
 deps
+pdo_enclave.config.xml

eservice/Makefile

@@ -69,7 +69,7 @@ $(SWIG_TARGET) : $(SWIG_FILES) $(ENCLAVE_LIB)
 
 build :
 	mkdir $@
-	cd $@ && cmake .. -G "Unix Makefiles"
+	cd $@ && cmake .. $(CMAKE_ARGS) -G "Unix Makefiles"
 
 install: $(EGG_FILE)
 	@ . $(abspath $(DSTDIR)/bin/activate) && \
@@ -85,10 +85,16 @@ test:
 	cd tests && pdo-test-request --no-ledger --iterations 10 --logfile $(TEST_LOG_FILE) --loglevel $(TEST_LOG_LEVEL)
 
 clean:
-	rm -f $(addprefix pdo/eservice/enclave/, pdo_enclave_internal.py pdo_enclave_internal_wrap.cpp)
-	rm -rf build deps dist *.egg-info
-	find . -iname '*.pyc' -delete
-	find . -iname '__pycache__' -delete
+	@ echo Remove build directory
+	@ if [ -d build ]; then \
+		make -C build clean; \
+		rm -rf build deps dist *.egg-info ; \
+	fi
+	@ echo clean swig files
+	@ rm -f $(addprefix pdo/eservice/enclave/, pdo_enclave_internal.py pdo_enclave_internal_wrap.cpp)
+	@ echo Clean up python caches
+	@ find . -iname '*.pyc' -delete
+	@ find . -iname '__pycache__' -delete
 
 print_enclave_files:
 	@echo $(ENCLAVE_FILES)

eservice/lib/libpdo_enclave/CMakeLists.txt

@@ -22,12 +22,13 @@ PROJECT(libpdo-enclave CXX C)
 FILE(GLOB PROJECT_HEADERS *.h)
 FILE(GLOB PROJECT_SOURCES *.cpp)
 FILE(GLOB PROJECT_EDL enclave.edl)
-FILE(GLOB PROJECT_CONFIG *.xml)
 FILE(GLOB PROJECT_LDS *.lds)
+SET(PROJECT_CONFIG ${CMAKE_CURRENT_SOURCE_DIR}/pdo_enclave.config.xml)
 
 SGX_EDGE_TRUSTED(${PROJECT_EDL} PROJECT_EDGE_SOURCES)
 SET (LIBPDO_ENCLAVE_EDL ${PROJECT_EDL} PARENT_SCOPE)
 
+
 # XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
 # Compile targets
 # XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
@@ -43,5 +44,8 @@ TARGET_LINK_LIBRARIES(${PROJECT_NAME} ${COMMON_TRUSTED_LIBS})
 TARGET_LINK_LIBRARIES(${PROJECT_NAME} -Wl,--end-group)
 
 SGX_PREPARE_TRUSTED_LINK(${PROJECT_NAME})
+
+# add dependency to ensure that enclave configuration file is created before post-build sgx-sign
+SGX_CONFIGURE_ENCLAVE(${PROJECT_NAME} ${PROJECT_CONFIG})
 SGX_SIGN_ENCLAVE(${PROJECT_NAME} ${PDO_SGX_KEY_ROOT}/enclave_code_sign.pem ${PROJECT_CONFIG})
 SGX_DEPLOY_FILES(${PROJECT_NAME} eservice)

eservice/lib/libpdo_enclave/pdo_enclave.config.xml → eservice/lib/libpdo_enclave/pdo_enclave.config.xml.in

@@ -23,7 +23,7 @@ limitations under the License.
   <ReservedMemExecutable>1</ReservedMemExecutable>
   <TCSNum>2</TCSNum>
   <TCSPolicy>1</TCSPolicy>
-  <DisableDebug>0</DisableDebug>
+  <DisableDebug>${DISABLE_DEBUG}</DisableDebug>
   <MiscSelect>0</MiscSelect>
   <MiscMask>0xFFFFFFFF</MiscMask>
 </EnclaveConfiguration>

eservice/pdo/eservice/enclave/enclave/enclave.cpp

@@ -432,17 +432,16 @@ namespace pdo {
                 Enclave::QuerySgxStatus();
 
                 sgx_launch_token_t token = { 0 };
-                int flags = SGX_DEBUG_FLAG;
-                pdo::error::ThrowSgxError((SGX_DEBUG_FLAG==0 ? SGX_ERROR_UNEXPECTED:SGX_SUCCESS),
-                    "SGX DEBUG flag is 0 (possible cause: wrong compile flags)");
+
+                pdo::logger::LogV(PDO_LOG_DEBUG, "LoadEnclave, SGX_DEBUG_FLAG: %d", SGX_DEBUG_FLAG);
 
                 // First attempt to load the enclave executable
                 sgx_status_t ret = SGX_SUCCESS;
-                ret = this->CallSgx([this, flags, &token] () {
+                ret = this->CallSgx([this, &token] () {
                         int updated = 0;
                         return sgx_create_enclave(
                             this->enclaveFilePath.c_str(),
-                            flags,
+                            SGX_DEBUG_FLAG,
                             &token,
                             &updated,
                             &this->enclaveId,

eservice/setup.py

@@ -36,11 +36,6 @@
 log_dir = os.path.join(install_root_dir, "logs")
 key_dir = os.path.join(install_root_dir, "keys")
 
-sgx_mode_env = os.environ.get('SGX_MODE', None)
-if not sgx_mode_env or (sgx_mode_env != "SIM" and sgx_mode_env != "HW"):
-    print("error: SGX_MODE value must be HW or SIM, current value is: ", sgx_mode_env)
-    sys.exit(2)
-
 data_files = [
     (bin_dir, [
         'bin/es-start.sh', 'bin/es-stop.sh', 'bin/es-status.sh',
@@ -59,7 +54,17 @@
 ## -----------------------------------------------------------------
 ## set up the contract enclave
 ## -----------------------------------------------------------------
-debug_flag = os.environ.get('PDO_DEBUG_BUILD',0)
+debug_flag_env = os.environ.get('PDO_DEBUG_BUILD', '0')
+if debug_flag_env not in ['0', '1'] :
+    print(f'error: PDO_DEBUG_BUILD must be 0 or 1, current value is {debug_flag_env}')
+    sys.exit(2)
+debug_flag = debug_flag_env == '1'
+
+sgx_mode_env = os.environ.get('SGX_MODE', 'SIM').upper()
+if sgx_mode_env not in ['SIM', 'HW'] :
+    print(f'error: SGX_MODE value must be HW or SIM, current value is {sgx_mode_env}')
+    sys.exit(2)
+sgx_simulator_flag = sgx_mode_env == 'SIM'
 
 module_path = 'pdo/eservice/enclave'
 module_src_path = os.path.join(script_dir, module_path)
@@ -99,16 +104,12 @@
     'lmdb'
 ]
 
-if sgx_mode_env == "HW":
-    libraries.append('sgx_urts')
-    libraries.append('sgx_uae_service')
-    SGX_SIMULATOR_value = '0'
-if sgx_mode_env == "SIM":
-    libraries.append('sgx_urts_sim')
-    libraries.append('sgx_uae_service_sim')
-    SGX_SIMULATOR_value = '1'
+if sgx_simulator_flag :
+    libraries += ['sgx_urts_sim', 'sgx_uae_service_sim']
+else :
+    libraries += ['sgx_urts', 'sgx_uae_service']
 
-libraries.append('sgx_usgxssl')
+libraries += ['sgx_usgxssl']
 
 module_files = [
     os.path.join(module_src_path, 'pdo_enclave_internal.i'),
@@ -128,10 +129,20 @@
 
 compile_defs = [
     ('_UNTRUSTED_', 1),
-    ('PDO_DEBUG_BUILD', debug_flag),
-    ('SGX_SIMULATOR', SGX_SIMULATOR_value)
+    ('PDO_DEBUG_BUILD', 1 if debug_flag else 0),
+    ('SGX_SIMULATOR', 1 if sgx_simulator_flag else 0),
 ]
 
+compile_undefs = []
+
+# When the debug flag (PDO_DEBUG_BUILD) is set, we set the EDEBUG define
+# This ensures that the SGX SDK in sgx_urts.h sets the SGX_DEBUG_FLAG to 1.
+# Otherwise the SDK sets it to 0.
+if debug_flag :
+    compile_defs += [('NDEBUG', 1), ('EDEBUG', 1)]
+else :
+    compile_undefs += ['NDEBUG', 'EDEBUG']
+
 swig_flags = ['-c++', '-threads']
 
 enclave_module = Extension(
@@ -143,7 +154,7 @@
     include_dirs = include_dirs,
     library_dirs = library_dirs,
     define_macros = compile_defs,
-    undef_macros = ['NDEBUG', 'EDEBUG']
+    undef_macros = compile_undefs,
     )
 
 ## -----------------------------------------------------------------

pservice/Makefile

@@ -72,17 +72,23 @@ $(SWIG_TARGET) : $(SWIG_FILES) $(ENCLAVE_LIB)
 
 build :
 	mkdir $@
-	cd $@ && cmake .. -G "Unix Makefiles"
+	cd $@ && cmake .. $(CMAKE_ARGS) -G "Unix Makefiles"
 
 install: $(EGG_FILE)
 	@ . $(abspath $(DSTDIR)/bin/activate) && \
 		python3 setup.py install
 
 clean:
-	rm -f $(addprefix pdo/pservice/enclave/, pdo_enclave_internal.py pdo_enclave_internal_wrap.cpp)
-	rm -rf build deps dist *.egg-info
-	find . -iname '*.pyc' -delete
-	find . -iname '__pycache__' -delete
+	@ echo Remove build directory
+	@ if [ -d build ]; then \
+		make -C build clean; \
+		rm -rf build deps dist *.egg-info ; \
+	fi
+	@ echo clean swig files
+	@ rm -f $(addprefix pdo/eservice/enclave/, pdo_enclave_internal.py pdo_enclave_internal_wrap.cpp)
+	@ echo Clean up python caches
+	@ find . -iname '*.pyc' -delete
+	@ find . -iname '__pycache__' -delete
 
 .phony : all
 .phony : clean