From bf098481c3f675e2e2f03b2ab5cc533000bfb469 Mon Sep 17 00:00:00 2001 From: Evgeny Gorodetskiy Date: Wed, 3 Jan 2024 17:34:09 +0300 Subject: [PATCH] Add experimental support of Apple's Metal Shader Converter instead of SPIRV-Cross --- Build/README.md | 47 +++++++------- CMake/MethaneShaders.cmake | 64 ++++++++++++++++--- CMakeLists.txt | 4 ++ Modules/Graphics/RHI/Metal/CMakeLists.txt | 8 +++ .../Include/Methane/Graphics/Metal/Shader.hh | 2 + .../Sources/Methane/Graphics/Metal/Shader.mm | 54 ++++++++++++++-- 6 files changed, 141 insertions(+), 38 deletions(-) diff --git a/Build/README.md b/Build/README.md index d724687c9..7dab387f3 100644 --- a/Build/README.md +++ b/Build/README.md @@ -191,29 +191,30 @@ Build options listed in table below can be used in cmake generator command line: cmake -G [Generator] ... -D[BUILD_OPTION_NAME]:BOOL=[ON|OFF] ``` -| Build Option Name | Initial Value | Default Preset | Profiling Preset | Description | -|-------------------------------------------------|-----------------------------------|-----------------------------------|----------------------------------|-------------------------------------------------------------------------------------| -| METHANE_GFX_VULKAN_ENABLED | OFF | ... | ... | Enable Vulkan graphics API instead of platform native API | -| METHANE_APPS_BUILD_ENABLED | ON | ON | ON | Enable applications build | -| METHANE_TESTS_BUILD_ENABLED | ON | ON | OFF | Enable tests build | -| METHANE_RHI_PIMPL_INLINE_ENABLED | ON (in Release) | ON (in Release) | ON | Enable RHI PIMPL implementation inlining | -| METHANE_PRECOMPILED_HEADERS_ENABLED | ON (not Apple) | ON (not Apple) | ON (not Apple) | Enable precompiled headers | -| METHANE_CHECKS_ENABLED | ON | ON | ON | Enable runtime checks of input arguments | -| METHANE_RUN_TESTS_DURING_BUILD | ON | OFF | OFF | Enable test auto-run after module build | -| METHANE_UNITY_BUILD_ENABLED | ON | ON | ON | Enable unity build speedup for some modules | -| METHANE_CODE_COVERAGE_ENABLED | OFF | OFF | OFF | Enable code coverage data collection with GCC and Clang | -| METHANE_SHADERS_CODEVIEW_ENABLED | OFF | ON | ON | Enable shaders code symbols viewing in debug tools | -| METHANE_OPEN_IMAGE_IO_ENABLED | OFF | OFF | OFF | Enable using OpenImageIO library for images loading | -| METHANE_COMMAND_DEBUG_GROUPS_ENABLED | OFF | ON | ON | Enable command list debug groups with frame markup | -| METHANE_LOGGING_ENABLED | OFF | OFF | OFF | Enable debug logging | -| METHANE_SCOPE_TIMERS_ENABLED | OFF | OFF | ON | Enable low-overhead profiling with scope-timers | -| METHANE_ITT_INSTRUMENTATION_ENABLED | OFF | ON | ON | Enable ITT instrumentation for trace capture with Intel GPA or VTune | -| METHANE_ITT_METADATA_ENABLED | OFF | OFF | ON | Enable ITT metadata for tasks and events like function source locations | -| METHANE_GPU_INSTRUMENTATION_ENABLED | OFF | OFF | ON | Enable GPU instrumentation to collect command list execution timings | -| METHANE_TRACY_PROFILING_ENABLED | OFF | OFF | ON | Enable realtime profiling with Tracy | -| METHANE_TRACY_PROFILING_ON_DEMAND | OFF | OFF | ON | Enable Tracy data collection on demand, after client connection | -| METHANE_MEMORY_SANITIZER_ENABLED | OFF | OFF | OFF | Enable memory address sanitizer in compiler and linker | -| METHANE_APPLE_CODE_SIGNING_ENABLED | OFF | OFF | OFF | Enable code signing on Apple platforms (requires APPLE_DEVELOPMENT_TEAM) | +| Build Option Name | Initial Value | Default Preset | Profiling Preset | Description | +|---------------------------------------------------|-----------------------------------|-----------------------------------|----------------------------------|-------------------------------------------------------------------------------------| +| METHANE_GFX_VULKAN_ENABLED | OFF | ... | ... | Enable Vulkan graphics API instead of platform native API | +| METHANE_APPS_BUILD_ENABLED | ON | ON | ON | Enable applications build | +| METHANE_TESTS_BUILD_ENABLED | ON | ON | OFF | Enable tests build | +| METHANE_RHI_PIMPL_INLINE_ENABLED | ON (in Release) | ON (in Release) | ON | Enable RHI PIMPL implementation inlining | +| METHANE_PRECOMPILED_HEADERS_ENABLED | ON (not Apple) | ON (not Apple) | ON (not Apple) | Enable precompiled headers | +| METHANE_CHECKS_ENABLED | ON | ON | ON | Enable runtime checks of input arguments | +| METHANE_RUN_TESTS_DURING_BUILD | ON | OFF | OFF | Enable test auto-run after module build | +| METHANE_UNITY_BUILD_ENABLED | ON | ON | ON | Enable unity build speedup for some modules | +| METHANE_CODE_COVERAGE_ENABLED | OFF | OFF | OFF | Enable code coverage data collection with GCC and Clang | +| METHANE_SHADERS_CODEVIEW_ENABLED | OFF | ON | ON | Enable shaders code symbols viewing in debug tools | +| METHANE_OPEN_IMAGE_IO_ENABLED | OFF | OFF | OFF | Enable using OpenImageIO library for images loading | +| METHANE_COMMAND_DEBUG_GROUPS_ENABLED | OFF | ON | ON | Enable command list debug groups with frame markup | +| METHANE_LOGGING_ENABLED | OFF | OFF | OFF | Enable debug logging | +| METHANE_SCOPE_TIMERS_ENABLED | OFF | OFF | ON | Enable low-overhead profiling with scope-timers | +| METHANE_ITT_INSTRUMENTATION_ENABLED | OFF | ON | ON | Enable ITT instrumentation for trace capture with Intel GPA or VTune | +| METHANE_ITT_METADATA_ENABLED | OFF | OFF | ON | Enable ITT metadata for tasks and events like function source locations | +| METHANE_GPU_INSTRUMENTATION_ENABLED | OFF | OFF | ON | Enable GPU instrumentation to collect command list execution timings | +| METHANE_TRACY_PROFILING_ENABLED | OFF | OFF | ON | Enable realtime profiling with Tracy | +| METHANE_TRACY_PROFILING_ON_DEMAND | OFF | OFF | ON | Enable Tracy data collection on demand, after client connection | +| METHANE_MEMORY_SANITIZER_ENABLED | OFF | OFF | OFF | Enable memory address sanitizer in compiler and linker | +| METHANE_APPLE_CODE_SIGNING_ENABLED | OFF | OFF | OFF | Enable code signing on Apple platforms (requires APPLE_DEVELOPMENT_TEAM) | +| METHANE_METAL_SHADER_CONVERTER_ENABLED | OFF | OFF | OFF | Enable Metal Shader Converter instead of SPIRV-Cross on Apple platforms | ### CMake Presets diff --git a/CMake/MethaneShaders.cmake b/CMake/MethaneShaders.cmake index 757342027..1e426ca24 100644 --- a/CMake/MethaneShaders.cmake +++ b/CMake/MethaneShaders.cmake @@ -73,9 +73,13 @@ endfunction() function(get_generated_shader_extension OUT_SHADER_EXT) if(METHANE_GFX_API EQUAL METHANE_GFX_DIRECTX) - set(${OUT_SHADER_EXT} "obj" PARENT_SCOPE) + set(${OUT_SHADER_EXT} "dxil" PARENT_SCOPE) elseif(METHANE_GFX_API EQUAL METHANE_GFX_METAL) - set(${OUT_SHADER_EXT} "metal" PARENT_SCOPE) + if (METHANE_METAL_SHADER_CONVERTER_ENABLED) + set(${OUT_SHADER_EXT} "dxil" PARENT_SCOPE) + else() + set(${OUT_SHADER_EXT} "metal" PARENT_SCOPE) + endif() elseif(METHANE_GFX_API EQUAL METHANE_GFX_VULKAN) set(${OUT_SHADER_EXT} "spirv" PARENT_SCOPE) endif() @@ -86,7 +90,6 @@ function(generate_metal_shaders_from_hlsl FOR_TARGET SHADERS_HLSL PROFILE_VER SH get_file_name(${SHADERS_HLSL} SHADERS_NAME) set(DXC_EXE "${DXC_BINARY_DIR}/dxc") - #set(SPIRV_GEN_EXE "${SPIRV_BINARY_DIR}/glslangValidator") set(SPIRV_CROSS_EXE "${SPIRV_BINARY_DIR}/spirv-cross") foreach(KEY_VALUE_STRING ${SHADER_TYPES}) @@ -118,7 +121,6 @@ function(generate_metal_shaders_from_hlsl FOR_TARGET SHADERS_HLSL PROFILE_VER SH DEPENDS "${SHADERS_HLSL}" COMMAND ${CMAKE_COMMAND} -E make_directory "${TARGET_SHADERS_DIR}" COMMAND ${DXC_EXE} -spirv -T ${SHADER_PROFILE} -E ${OLD_ENTRY_POINT} ${SHADER_DEFINITION_ARGUMENTS} "${SHADERS_HLSL}" -Fo "${SHADER_SPIRV_PATH}" - #COMMAND ${SPIRV_GEN_EXE} --hlsl-iomap -S ${SHADER_TYPE} -e ${OLD_ENTRY_POINT} ${SHADER_DEFINITION_ARGUMENTS} -o "${SHADER_SPIRV_PATH}" -V -D "${SHADERS_HLSL}" COMMAND ${SPIRV_CROSS_EXE} --msl --msl-version 020101 --msl-decoration-binding --rename-entry-point ${OLD_ENTRY_POINT} ${NEW_ENTRY_POINT} ${SHADER_TYPE} --output "${SHADER_METAL_PATH}" "${SHADER_SPIRV_PATH}" ) @@ -196,6 +198,43 @@ function(compile_metal_shaders_to_library FOR_TARGET SDK METAL_SHADERS METAL_LIB add_dependencies(${FOR_TARGET} ${METAL_LIB_TARGET}) endfunction() +function(compile_dxil_to_metal_library FOR_TARGET COMPILE_SHADER_TARGETS COMPILED_SHADER_BINARIES LIBRARY_NAME OUT_METAL_LIBRARIES) + + foreach(DXIL_SHADER_BINARY ${COMPILED_SHADER_BINARIES}) + if (METAL_SHADER_CONV_COMMAND) + list(APPEND METAL_SHADER_CONV_COMMAND "&&") + endif() + get_target_shaders_dir(${FOR_TARGET} TARGET_SHADERS_DIR) + get_file_name(${DXIL_SHADER_BINARY} SHADER_METAL_LIBRARY_NAME) + set(SHADER_METAL_LIBRARY "${TARGET_SHADERS_DIR}/${SHADER_METAL_LIBRARY_NAME}.metallib") + list(APPEND METAL_SHADER_CONV_COMMAND "metal-shaderconverter" -o "${SHADER_METAL_LIBRARY}" "${DXIL_SHADER_BINARY}") + list(APPEND SHADER_METAL_LIBRARIES "${SHADER_METAL_LIBRARY}") + endforeach () + + set(METAL_LIB_TARGET ${FOR_TARGET}_CompileMetalLibrary_${LIBRARY_NAME}) + add_custom_target(${METAL_LIB_TARGET} + COMMENT "Convert compiled DXIL to Metal libraries for application " ${TARGET} + BYPRODUCTS "${SHADER_METAL_LIBRARIES}" + DEPENDS "${COMPILED_SHADER_BINARIES}" + COMMAND ${METAL_SHADER_CONV_COMMAND} + ) + + set_target_properties(${METAL_LIB_TARGET} + PROPERTIES + FOLDER "Build/${FOR_TARGET}/Shaders" + ) + + set_target_properties(${FOR_TARGET} + PROPERTIES + METAL_LIB_TARGET ${METAL_LIB_TARGET} + ) + + add_dependencies(${METAL_LIB_TARGET} ${COMPILE_SHADER_TARGETS}) + add_dependencies(${FOR_TARGET} ${METAL_LIB_TARGET}) + + set(${OUT_METAL_LIBRARIES} "${SHADER_METAL_LIBRARIES}" PARENT_SCOPE) +endfunction() + function(compile_hlsl_shaders FOR_TARGET SHADERS_HLSL PROFILE_VER SHADER_TYPES OUT_COMPILED_SHADER_BINARIES OUT_COMPILE_SHADER_TARGETS) get_target_shaders_dir(${FOR_TARGET} TARGET_SHADERS_DIR) get_file_name(${SHADERS_HLSL} SHADERS_NAME) @@ -311,15 +350,24 @@ function(add_methane_shaders_source) set_property(TARGET ${SHADERS_TARGET} APPEND PROPERTY COMPILED_SHADER_BINARIES ${COMPILED_SHADER_BINARIES}) set_property(TARGET ${SHADERS_TARGET} APPEND PROPERTY COMPILE_SHADER_TARGETS ${COMPILE_SHADER_TARGETS}) - elseif(METHANE_GFX_API EQUAL METHANE_GFX_METAL) + elseif (METHANE_GFX_API EQUAL METHANE_GFX_METAL) set(SHADERS_METAL) # init with empty list get_metal_library(${SHADERS_TARGET} ${SHADERS_SOURCE_PATH} METAL_LIBRARY) get_apple_sdk(SDK_NAME) - generate_metal_shaders_from_hlsl(${SHADERS_TARGET} "${SHADERS_SOURCE_PATH}" "${SHADERS_VERSION}" "${SHADERS_TYPES}" SHADERS_METAL GENERATE_METAL_TARGETS) - compile_metal_shaders_to_library(${SHADERS_TARGET} "${SDK_NAME}" "${SHADERS_METAL}" "${METAL_LIBRARY}") + if (METHANE_METAL_SHADER_CONVERTER_ENABLED) + # Use Apple's Metal Shader Converter to compile from DXIL directly to Metal library + get_file_name(${METAL_LIBRARY} LIBRARY_NAME) + compile_hlsl_shaders(${SHADERS_TARGET} "${SHADERS_SOURCE_PATH}" "${SHADERS_VERSION}" "${SHADERS_TYPES}" COMPILED_SHADER_BINARIES COMPILE_SHADER_TARGETS) + compile_dxil_to_metal_library(${SHADERS_TARGET} "${COMPILE_SHADER_TARGETS}" "${COMPILED_SHADER_BINARIES}" "${LIBRARY_NAME}" METAL_LIBRARIES) + set_property(TARGET ${SHADERS_TARGET} APPEND PROPERTY METAL_LIBRARIES ${METAL_LIBRARIES}) + else() + # Use SPIRV-Cross to convert compiled HLSL as SPIRV to Metal shader sources and then compile to Metal library + generate_metal_shaders_from_hlsl(${SHADERS_TARGET} "${SHADERS_SOURCE_PATH}" "${SHADERS_VERSION}" "${SHADERS_TYPES}" SHADERS_METAL GENERATE_METAL_TARGETS) + compile_metal_shaders_to_library(${SHADERS_TARGET} "${SDK_NAME}" "${SHADERS_METAL}" "${METAL_LIBRARY}") + set_property(TARGET ${SHADERS_TARGET} APPEND PROPERTY METAL_LIBRARIES ${METAL_LIBRARY}) + endif() set_property(TARGET ${SHADERS_TARGET} APPEND PROPERTY METAL_SOURCES ${SHADERS_METAL}) - set_property(TARGET ${SHADERS_TARGET} APPEND PROPERTY METAL_LIBRARIES ${METAL_LIBRARY}) set_property(TARGET ${SHADERS_TARGET} APPEND PROPERTY GENERATE_METAL_TARGETS ${GENERATE_METAL_TARGETS}) endif() diff --git a/CMakeLists.txt b/CMakeLists.txt index d69e4659d..683f0ec34 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -68,6 +68,10 @@ option(METHANE_CODE_COVERAGE_ENABLED "Enable code coverage data collectio option(METHANE_SHADERS_CODEVIEW_ENABLED "Enable shaders code symbols viewing in debug tools" OFF) option(METHANE_OPEN_IMAGE_IO_ENABLED "Enable using OpenImageIO library for images loading" OFF) +if(APPLE) + option(METHANE_METAL_SHADER_CONVERTER_ENABLED "Enable Apple Metal Shader Converter instead of SPIRV-Cross" OFF) +endif() + # Profiling and instrumentation configuration option(METHANE_COMMAND_DEBUG_GROUPS_ENABLED "Enable command list debug groups with frame markup" OFF) option(METHANE_LOGGING_ENABLED "Enable debug logging" OFF) diff --git a/Modules/Graphics/RHI/Metal/CMakeLists.txt b/Modules/Graphics/RHI/Metal/CMakeLists.txt index c42d2a625..a20b3eac8 100644 --- a/Modules/Graphics/RHI/Metal/CMakeLists.txt +++ b/Modules/Graphics/RHI/Metal/CMakeLists.txt @@ -115,3 +115,11 @@ set_source_files_properties( SKIP_PRECOMPILE_HEADERS ON SKIP_UNITY_BUILD_INCLUSION ON ) + +if (METHANE_METAL_SHADER_CONVERTER_ENABLED) + set_source_files_properties( + ${SOURCES_DIR}/Shader.mm + PROPERTIES + COMPILE_FLAGS "-DMETAL_LIBRARY_SPLIT_BY_SHADER_ENTRY_FUNCTION" + ) +endif() \ No newline at end of file diff --git a/Modules/Graphics/RHI/Metal/Include/Methane/Graphics/Metal/Shader.hh b/Modules/Graphics/RHI/Metal/Include/Methane/Graphics/Metal/Shader.hh index be697282e..c61e727b5 100644 --- a/Modules/Graphics/RHI/Metal/Include/Methane/Graphics/Metal/Shader.hh +++ b/Modules/Graphics/RHI/Metal/Include/Methane/Graphics/Metal/Shader.hh @@ -49,6 +49,8 @@ public: private: const IContext& GetMetalContext() const noexcept; + static id GetMetalLibraryFunction(const IContext& context, const Rhi::ShaderSettings& settings); + id m_mtl_function; NSArray>* m_mtl_bindings = nil; }; diff --git a/Modules/Graphics/RHI/Metal/Sources/Methane/Graphics/Metal/Shader.mm b/Modules/Graphics/RHI/Metal/Sources/Methane/Graphics/Metal/Shader.mm index 9b1fd3558..1b46396bc 100644 --- a/Modules/Graphics/RHI/Metal/Sources/Methane/Graphics/Metal/Shader.mm +++ b/Modules/Graphics/RHI/Metal/Sources/Methane/Graphics/Metal/Shader.mm @@ -74,6 +74,7 @@ static MTLVertexStepFunction GetVertexStepFunction(StepType step_type) [[nodiscard]] static uint32_t GetBindingArrayLength(id mtl_binding) { + META_FUNCTION_TASK(); if (mtl_binding.type != MTLBindingTypeTexture) return 1U; @@ -81,6 +82,50 @@ static uint32_t GetBindingArrayLength(id mtl_binding) return static_cast(mtl_texture_binding.arrayLength); } +[[nodiscard]] +id Shader::GetMetalLibraryFunction(const IContext& context, const Rhi::ShaderSettings& settings) +{ + META_FUNCTION_TASK(); + const std::string compiled_entry_function_name = Base::Shader::GetCompiledEntryFunctionName(settings); + +#ifdef METAL_LIBRARY_SPLIT_BY_SHADER_ENTRY_FUNCTION + std::string_view shader_library_name = compiled_entry_function_name; +#else + std::string_view shader_library_name = settings.entry_function.file_name; +#endif + + const Ptr& program_library_ptr = context.GetMetalLibrary(shader_library_name); + META_CHECK_ARG_NOT_NULL(program_library_ptr); + id mtl_shader_library = program_library_ptr->GetNativeLibrary(); + +#ifdef METAL_LIBRARY_SPLIT_BY_SHADER_ENTRY_FUNCTION + NSString* ns_shader_function_name = mtl_shader_library.functionNames.firstObject; +#else + NSString* ns_shader_function_name = Methane::MacOS::ConvertToNsString(compiled_entry_function_name); +#endif + + return [mtl_shader_library newFunctionWithName: ns_shader_function_name]; +} + +[[nodiscard]] +static std::string GetAttributeName(NSString* ns_reflect_attrib_name) +{ + META_FUNCTION_TASK(); + + // Regex matching prefix of the input attributes "in_var_..." + static const std::regex s_attr_prefix_regex("^in_var_"); + + // Regex matching suffix of the input attributes "...123" + static const std::regex s_attr_suffix_regex("\\d+$"); + + std::string attrib_name = MacOS::ConvertFromNsString(ns_reflect_attrib_name); + attrib_name = std::regex_replace(attrib_name, s_attr_prefix_regex, ""); + attrib_name = std::regex_replace(attrib_name, s_attr_suffix_regex, ""); + + std::transform(attrib_name.begin(), attrib_name.end(), attrib_name.begin(), ::toupper); + return attrib_name; +} + #ifndef NDEBUG [[nodiscard]] @@ -114,8 +159,7 @@ static uint32_t GetBindingArrayLength(id mtl_binding) Shader::Shader(Rhi::ShaderType shader_type, const Base::Context& context, const Settings& settings) : Base::Shader(shader_type, context, settings) - , m_mtl_function([dynamic_cast(context).GetMetalLibrary(settings.entry_function.file_name)->GetNativeLibrary() - newFunctionWithName: Methane::MacOS::ConvertToNsString(GetCompiledEntryFunctionName())]) + , m_mtl_function(GetMetalLibraryFunction(dynamic_cast(context), settings)) { META_FUNCTION_TASK(); META_CHECK_ARG_NOT_NULL_DESCR(m_mtl_function, "failed to initialize Metal shader function by name '{}'", GetCompiledEntryFunctionName()); @@ -184,10 +228,6 @@ static uint32_t GetBindingArrayLength(id mtl_binding) MTLVertexDescriptor* Shader::GetNativeVertexDescriptor(const Program& program) const { META_FUNCTION_TASK(); - - // Regex matching prefix of the input attributes "in_var_" - static const std::regex s_attr_suffix_regex("^in_var_"); - MTLVertexDescriptor* mtl_vertex_desc = [[MTLVertexDescriptor alloc] init]; [mtl_vertex_desc reset]; @@ -198,7 +238,7 @@ static uint32_t GetBindingArrayLength(id mtl_binding) continue; const MTLVertexFormat mtl_vertex_format = TypeConverter::MetalDataTypeToVertexFormat(mtl_vertex_attrib.attributeType); - const std::string attrib_name = std::regex_replace(MacOS::ConvertFromNsString(mtl_vertex_attrib.name), s_attr_suffix_regex, ""); + const std::string attrib_name = GetAttributeName(mtl_vertex_attrib.name); const uint32_t attrib_size = TypeConverter::ByteSizeOfVertexFormat(mtl_vertex_format); const uint32_t attrib_slot = GetProgramInputBufferIndexByArgumentSemantic(program, attrib_name);