feat: Add API for building attribute getter free functions. (#1704)

Adds a new RendererServices API build_attribute_getter which allows for building custom free functions that provide values for attributes, effectively replacing both get_attribute and get_array_attribute.

This new API is invoked at material compilation time to allow render developers to provide specialized functions capable of taking advantage of known compile time information. This will allow for many optimization opportunities, such as replacing run-time map lookups with direct memory reads from known locations, remove brancing required for checking if derivatives are needed, branching on type-conversions if required, etc.

This PR currently implements this approach for attributes, a future PR could do the same for user-data.
Tests

TestShade provides an example implementation showing how this compile time information can be used to select an appropriate function to use as the attribute provider, and how to configure the function signature.

---------

Signed-off-by: Curtis Black <curtis.w.black@gmail.com>

CMakeLists.txt

@@ -289,6 +289,15 @@ configure_package_config_file ("${PROJECT_SOURCE_DIR}/src/cmake/Config.cmake.in"
 install (FILES "${OSL_PROJECT_CONFIG}" "${OSL_VERSION_CONFIG}"
         DESTINATION "${OSL_CONFIG_INSTALL_DIR}")
 
+install (FILES src/cmake/llvm_macros.cmake DESTINATION cmake)
+
+set (PERMISSION_FLAGS OWNER_EXECUTE OWNER_READ OWNER_WRITE
+                      GROUP_EXECUTE GROUP_READ
+                      WORLD_EXECUTE WORLD_READ)
+install (FILES src/build-scripts/serialize-bc.py
+         DESTINATION build-scripts
+         PERMISSIONS ${PERMISSION_FLAGS})
+
 # install targets files
 install (EXPORT OSL_EXPORTED_TARGETS
         DESTINATION ${OSL_CONFIG_INSTALL_DIR}

src/cmake/llvm_macros.cmake

@@ -2,8 +2,10 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # https://github.com/AcademySoftwareFoundation/OpenShadingLanguage
 
+# TODO: Use CMAKE_CURRENT_FUNCTION_LIST_DIR in cmake-3.17
+set(_THIS_MODULE_BASE_DIR "${CMAKE_CURRENT_LIST_DIR}")
 
-function ( EMBED_LLVM_BITCODE_IN_CPP src_list suffix output_name list_to_append_cpp extra_clang_args)
+function ( EMBED_LLVM_BITCODE_IN_CPP src_list suffix output_name list_to_append_cpp extra_clang_args include_dirs)
 
     if (VERBOSE)
         message (STATUS "EMBED_LLVM_BITCODE_IN_CPP src_list=${src_list}")
@@ -75,12 +77,8 @@ function ( EMBED_LLVM_BITCODE_IN_CPP src_list suffix output_name list_to_append_
         #    endif ()
         #endif ()
 
-        list (TRANSFORM IMATH_INCLUDES PREPEND -I
-            OUTPUT_VARIABLE ALL_IMATH_INCLUDES)
-        list (TRANSFORM OPENEXR_INCLUDES PREPEND -I
-            OUTPUT_VARIABLE ALL_OPENEXR_INCLUDES)
-        list (TRANSFORM OpenImageIO_INCLUDES PREPEND -I
-            OUTPUT_VARIABLE ALL_OpenImageIO_INCLUDES)
+        list (TRANSFORM include_dirs PREPEND -I
+            OUTPUT_VARIABLE ALL_INCLUDE_DIRS)
 
         if (${LLVM_VERSION} VERSION_GREATER_EQUAL 15.0)
             # Until we fully support opaque pointers, we need to disable
@@ -93,13 +91,7 @@ function ( EMBED_LLVM_BITCODE_IN_CPP src_list suffix output_name list_to_append_
         add_custom_command ( OUTPUT ${src_bc}
         COMMAND ${LLVM_BC_GENERATOR}
             ${LLVM_COMPILE_FLAGS}
-            "-I${CMAKE_CURRENT_SOURCE_DIR}"
-            "-I${CMAKE_SOURCE_DIR}/src/include"
-            "-I${CMAKE_BINARY_DIR}/include"
-            ${ALL_OpenImageIO_INCLUDES}
-            ${ALL_IMATH_INCLUDES}
-            #"-isystem ${Boost_INCLUDE_DIRS}" #Does not pick up usr installed boost/thread/tss.hpp for oslexec_pvt.h
-            "-I${Boost_INCLUDE_DIRS}"
+            ${ALL_INCLUDE_DIRS}
             -DOSL_COMPILING_TO_BITCODE=1
             -Wno-deprecated-register
             # the following 2 warnings can be restored when all 3rd parties have fixed their export macros
@@ -131,9 +123,9 @@ function ( EMBED_LLVM_BITCODE_IN_CPP src_list suffix output_name list_to_append_
      # Serialize the linked bitcode into a CPP file 
     set ( src_bc_cpp "${CMAKE_CURRENT_BINARY_DIR}/${output_name}.bc.cpp" )
     add_custom_command ( OUTPUT ${src_bc_cpp}
-        COMMAND ${Python_EXECUTABLE} "${CMAKE_SOURCE_DIR}/src/build-scripts/serialize-bc.py"
+        COMMAND ${Python_EXECUTABLE} "${_THIS_MODULE_BASE_DIR}/../build-scripts/serialize-bc.py"
             ${linked_src_bc} ${src_bc_cpp} ${output_name}
-        DEPENDS "${CMAKE_SOURCE_DIR}/src/build-scripts/serialize-bc.py" ${linked_src_bc}
+        DEPENDS "${_THIS_MODULE_BASE_DIR}/../build-scripts/serialize-bc.py" ${linked_src_bc}
         ${exec_headers} ${PROJECT_PUBLIC_HEADERS}
         WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" )
 

src/include/OSL/llvm_util.h

@@ -665,11 +665,17 @@ class OSLEXECPUBLIC LLVM_Util {
     /// Return an llvm::Constant holding the given floating point constant.
     llvm::Constant* constant(float f);
 
+    /// Return an llvm::Constant holding the given floating point constant.
+    llvm::Constant* constant64(double f);
+
     /// Return an llvm::Constant holding the given integer constant.
-    llvm::Constant* constant(int i);
+    llvm::Constant* constant(int32_t i);
+    llvm::Constant* constant(uint32_t i);
 
     /// Return an llvm::Constant holding the given integer constant.
-    llvm::Constant* constant8(int i);
+    llvm::Constant* constant8(int8_t i);
+    llvm::Constant* constant8(uint8_t i);
+    llvm::Constant* constant16(int16_t i);
     llvm::Constant* constant16(uint16_t i);
     llvm::Constant* constant64(uint64_t i);
     llvm::Constant* constant128(uint64_t i);

src/include/OSL/oslconfig.h.in

@@ -266,6 +266,20 @@ ustringrep_from(string_view s)
 
 
 
+using TypeDesc_pod = int64_t;
+
+static_assert(sizeof(TypeDesc_pod) == sizeof(TypeDesc),
+              "TypeDesc size differs from its POD counterpart");
+
+/// Convenience function to convert to a TypeDesc.
+inline TypeDesc
+TypeDesc_from(TypeDesc_pod type)
+{
+    return OSL::bitcast<OSL::TypeDesc>(type);
+}
+
+
+
 // N.B. SymArena is not really "configuration", but we cram it here for
 // lack of a better home.
 

src/include/OSL/oslexec.h

@@ -1033,6 +1033,18 @@ class OSLEXECPUBLIC ShadingSystem {
     void add_symlocs(cspan<SymLocationDesc> symlocs);
     void add_symlocs(ShaderGroup* group, cspan<SymLocationDesc> symlocs);
 
+    // Find the SymLocationDesc for this named param, returning its pointer
+    // or nullptr if that name is not found.
+    const SymLocationDesc* find_symloc(ustring name) const;
+    const SymLocationDesc* find_symloc(ShaderGroup* group, ustring name) const;
+
+    // Find the SymLocationDesc for this named param but only if it matches
+    // the arena type, returning its pointer or nullptr if that name is not
+    // found.
+    const SymLocationDesc* find_symloc(ustring name, SymArena arena) const;
+    const SymLocationDesc* find_symloc(ShaderGroup* group, ustring name,
+                                       SymArena arena) const;
+
     /// Ensure that the group has been optimized and optionally JITed. The ctx pointer
     /// supplies a ShadingContext to use.
     void optimize_group(ShaderGroup* group, ShadingContext* ctx,

src/include/OSL/rendererservices.h

@@ -7,6 +7,7 @@
 
 #include <OSL/encodedtypes.h>
 #include <OSL/oslconfig.h>
+#include <OSL/variant.h>
 
 
 OSL_NAMESPACE_ENTER
@@ -15,6 +16,7 @@ class RendererServices;
 template<int WidthT> class BatchedRendererServices;
 class ShadingContext;
 struct ShaderGlobals;
+class ShaderGroup;
 
 // Tags for polymorphic dispatch
 template<int SimdWidthT> class WidthOf {
@@ -30,6 +32,20 @@ typedef const void* TransformationPtr;
 typedef void (*PrepareClosureFunc)(RendererServices*, int id, void* data);
 typedef void (*SetupClosureFunc)(RendererServices*, int id, void* data);
 
+enum class AttributeSpecBuiltinArg {
+    ShaderGlobalsPointer,  // void* (TODO: ideally ShaderGlobals*)
+    ShadeIndex,            // int
+    Derivatives,           // bool
+    Type,                  // TypeDesc_pod
+    ArrayIndex,            // int, Always zero for non-indexed array lookups.
+    IsArrayLookup,         // bool
+    ObjectName,            // const char* (TODO: change to ustringhash)
+    AttributeName,         // const char* (TODO: change to ustringhash)
+};
+
+using AttributeSpecArg    = ArgVariant<AttributeSpecBuiltinArg>;
+using AttributeGetterSpec = FunctionSpec<AttributeSpecArg>;
+
 // Turn off warnings about unused params for this file, since we have lots
 // of declarations with stub function bodies.
 OSL_PRAGMA_WARNING_PUSH
@@ -51,7 +67,8 @@ class OSLEXECPUBLIC RendererServices {
 
     /// Given the name of a 'feature', return whether this RendererServices
     /// supports it. Feature names include:
-    ///    <none>
+    ///    "OptiX"
+    ///    "build_attribute_getter"
     ///
     /// This allows some customization of JIT generated code based on the
     /// facilities and features of a particular renderer. It also allows
@@ -189,6 +206,47 @@ class OSLEXECPUBLIC RendererServices {
                          const EncodedType* arg_types, uint32_t arg_values_size,
                          uint8_t* arg_values);
 
+    /// Builds a free function to provide a value for a given attribute.
+    /// This occurs at shader compile time, not at execution time.
+    ///
+    /// @param group
+    ///     The shader group currently requesting the attribute.
+    ///
+    /// @param is_object_lookup
+    ///     True if an object name was specified, even if the value is not
+    ///     known at compile time.
+    ///
+    /// @param object_name
+    ///     The object name, or nullptr if the value is not specified or it
+    ///     is not known at compile time.
+    ///
+    /// @param attribute_name
+    ///     The attribute name, or nullptr if the value is not known at
+    ///     compile time.
+    ///
+    /// @param is_array_lookup
+    ///     True if the attribute lookup provides an index.
+    ///
+    /// @param array_index
+    ///     The array index, or nullptr if the value is not specified or it
+    ///     is not known at compile time.
+    ///
+    /// @param type
+    ///     The type of the value being requested.
+    ///
+    /// @param derivatives
+    ///     True if derivatives are also being requested.
+    ///
+    /// @param spec
+    ///     The built attribute getter. An empty function name is interpreted
+    ///     as a missing attribute.
+    ///
+    virtual void
+    build_attribute_getter(const ShaderGroup& group, bool is_object_lookup,
+                           const ustring* object_name,
+                           const ustring* attribute_name, bool is_array_lookup,
+                           const int* array_index, TypeDesc type,
+                           bool derivatives, AttributeGetterSpec& spec);
 
     /// Get the named attribute from the renderer and if found then
     /// write it into 'val'.  Otherwise, return false.  If no object is