NativeAOT-LLVM: RyuJIT IR -> LLVM for improved performance

docs/workflow/building/coreclr/nativeaot.md

@@ -11,11 +11,27 @@ The Native AOT toolchain can be currently built for Linux, macOS and Windows x64
    - Run `dotnet publish --packages pkg -r [win-x64|linux-x64|osx-64] -c [Debug|Release]` to publish your project. `--packages pkg` option restores the package into a local directory that is easy to cleanup once you are done. It avoids polluting the global nuget cache with your locally built dev package.
 
 ## Building for Web Assembly
+- This branch contains a version of the WebAssembly compiler that creates LLVM from the clrjit to take advantage of RyuJits optimisations.  It goes from RyuJIT IR -> LLVM instead of the NativeAOT-LLVM branch way of CIL -> LLVM.
+- It does not work, yet or maybe never.
 - Currently only tested on Windows
+- Download the LLVM 11.0.0 source from https://github.com/llvm/llvm-project/releases/download/llvmorg-11.0.0/llvm-11.0.0.src.tar.xz
+- Extract and create a subdirectory in the llvm-11.0.0.src folder called build.  cd to this build folder
+- Configure the LLVM source to use the same runtime as clrjit `cmake -G "Visual Studio 16 2019" -DCMAKE_BUILD_TYPE=Debug -D LLVM_USE_CRT_DEBUG=MTd ..`
+- Build LLVM either from the command line (`build`) or from VS 2019.  You only really need to build the LLVMCore and LLVMBitWriter projects which takes less time than the 400 odd projects when building all.  This will save some time.
+- set the enviroment variable LLVM_CMAKE_CONFIG to locate the LLVM config, e.g.  `set LLVM_CMAKE_CONFIG=E:/llvm11/llvm-11.0.0.src/build/lib/cmake/llvm` .   This location should contain the file `LLVMConfig.cmake`
+- Build the x64 libraries and compiler as per the Building section.
 - Run `build nativeaot+libs+nativeaot.packages -rc [Debug|Release] -lc [Debug|Release] -a wasm -os Browser -runtimeFlavor CoreCLR`
+- The compiler can now be debugged with the Wasm clrjit.  Load the clrjit_browser_wasm32_x64.vcxproj which can be found in artifacts\obj\coreclr\windows.x64.Debug\jit
+- Run Ilc with a .rsp file as normal for Web assembly, e.g. if you build the WebAssembly tests you can use artifacts\tests\coreclr\Browser.wasm.Debug\nativeaot\SmokeTests\HelloWasm\HelloWasm\native\HelloWasm.ilc.rsp
 - Add the package directory to your `nuget.config` as above.
 - Run `dotnet publish -r browser-wasm -c [Debug|Release] /p:Platform=wasm` to publish.
 
+- To work on the clr jit for LLVM:
+- Open the Ilc solution and add the clr jit project `clrjit_browser_wasm32_x64.vcxproj` from `artifacts\obj\coreclr\windows.x64.Debug\jit`
+- In the project properties General section, change the output folder to the full path for `artifacts\bin\coreclr\windows.x64.Debug\ilc\net5.0` e.g. `E:\GitHub\runtimelab\artifacts\bin\coreclr\windows.x64.Debug\ilc\net5.0`
+- Build `clrjit_browser_wasm32_x64` project and you should now be able to change and put breakpoints in the c++ code.
+
+
 ## Visual Studio Solutions
 
 The repository has a number of Visual Studio Solutions files (`*.sln`) that are useful for editing parts of the repository. Build the repo from command line first before building using the solution files. Remember to select the appropriate configuration that you built. By default, `build.cmd` builds Debug x64 and so `Debug` and `x64` must be selected in the solution build configuration drop downs.
@@ -40,6 +56,7 @@ The workflow looks like this:
 - Open the ilc.sln solution described above. This solution contains the compiler, but also an unrelated project named "repro". This repro project is a small Hello World. You can place any piece of C# you would like to compile in it. Building the project will compile the source code into IL, but also generate a response file that is suitable to pass to the AOT compiler.
 - Make sure you set the solution configuration in VS to the configuration you just built (e.g. x64 Debug).
 - In the ILCompiler project properties, on the Debug tab, set the "Application arguments" to the generated response file. This will be a file such as "C:\runtimelab\artifacts\bin\repro\x64\Debug\compile-with-Release-libs.rsp". Prefix the path to the file with "@" to indicate this is a response file so that the "Application arguments" field looks like "@some\path\to\file.rsp".
+- For WebAssembly, edit the .rsp file and 
 - Build & run ILCompiler using **F5**. This will compile the repro project into an `.obj` file. You can debug the compiler and set breakpoints in it at this point.
 - The last step is linking the file into an executable so that we can launch the result of the AOT compilation.
 - Open the src\coreclr\tools\aot\ILCompiler\reproNative\reproNative.vcxproj project in Visual Studio. This project is configured to pick up the `.obj` file we just compiled and link it with the rest of the runtime.

eng/common/build.ps1

@@ -143,6 +143,7 @@ try {
     }
     if ($properties.Contains('/p:TargetArchitecture=wasm') -and $runtimeFlavor -eq "CoreCLR") {
       . $PSScriptRoot\..\..\wasm-tools\emsdk\emsdk_env.ps1
+      $Env:LLVM_CMAKE_CONFIG = "$PSScriptRoot\..\..\wasm-tools\llvm-11.0.0.src\build\lib\cmake\llvm"
     }
     $nodeReuse = $false
   }

eng/native/configurecompiler.cmake

@@ -530,6 +530,14 @@ if (MSVC)
   add_compile_options($<$<COMPILE_LANGUAGE:C,CXX>:/wd4291>)
   add_compile_options($<$<COMPILE_LANGUAGE:C,CXX>:/wd5105>)
 
+  # TODO: if for LLVM
+  add_compile_options($<$<COMPILE_LANGUAGE:C,CXX>:/wd4244>)
+  add_compile_options($<$<COMPILE_LANGUAGE:C,CXX>:/wd4267>)
+  add_compile_options($<$<COMPILE_LANGUAGE:C,CXX>:/wd4141>)
+  add_compile_options($<$<COMPILE_LANGUAGE:C,CXX>:/wd4310>)
+  add_compile_options($<$<COMPILE_LANGUAGE:C,CXX>:/wd4624>) # destructor was implicitly defined as deleted
+  add_compile_options($<$<COMPILE_LANGUAGE:C,CXX>:/wd4324>) # structure was padded due to alignment specifier
+
   # Treat Warnings as Errors:
   # 4007: 'main' : must be __cdecl.
   # 4013: 'function' undefined - assuming extern returning int.

eng/pipelines/common/global-build-job.yml

@@ -122,9 +122,11 @@ jobs:
         displayName: Disk Usage before Build
 
     - ${{ if and(eq(parameters.runtimeFlavor, 'coreclr'), and(eq(parameters.osGroup, 'windows'), eq(parameters.platform, 'Browser_wasm'))) }}:
-      # Install Wasm dependencies: emscripten
+      # Install Wasm dependencies: emscripten, LLVM
       - script: call $(Build.SourcesDirectory)/eng/pipelines/runtimelab/install-emscripten.cmd $(Build.SourcesDirectory)\wasm-tools
         displayName: Install/activate emscripten
+      - script: call $(Build.SourcesDirectory)/eng/pipelines/runtimelab/install-llvm.cmd $(Build.SourcesDirectory)\wasm-tools $(Build.SourcesDirectory)
+        displayName: Install/build LLVM
 
     # Build
     - script: $(Build.SourcesDirectory)$(dir)build$(scriptExt) -ci -arch ${{ parameters.archType }} $(_osParameter) ${{ parameters.buildArgs }} $(_officialBuildParameter) $(_crossBuildPropertyArg) $(_cxx11Parameter) $(_richCodeNavigationParam)

eng/pipelines/runtimelab/install-llvm.cmd

@@ -0,0 +1,24 @@
+mkdir "%1" 2>nul
+cd /D "%1"
+
+set RepoRoot=%2\
+
+set
+:: Set CMakePath by evaluating the output from set-cmake-path.ps1
+call "%RepoRoot%src\coreclr\setup_vs_tools.cmd" || exit /b 1
+for /f "delims=" %%a in ('powershell -NoProfile -ExecutionPolicy ByPass "& ""%RepoRoot%eng\native\set-cmake-path.ps1"""') do %%a
+echo Using CMake at "%CMakePath%"
+
+set
+
+powershell -NoProfile -NoLogo -ExecutionPolicy ByPass -command "& """%~dp0install-llvm.ps1""" %*"
+if %errorlevel% NEQ 0 goto fail
+
+echo setting LLVM_CMAKE_CONFIG to %1\llvm-11.0.0.src\build
+echo "##vso[task.setvariable variable=LLVM_CMAKE_CONFIG]%1\llvm-11.0.0.src\build"
+
+exit /b 0
+
+fail:
+echo "Failed to install llvm"
+exit /b 1

eng/pipelines/runtimelab/install-llvm.ps1

@@ -0,0 +1,31 @@
+# LLVM is supplied in a gz file which Windows doesn't native understand, so we need gz to unpack it - TODO this is liable to fail randomly when a new version comes out and the version number changes
+Invoke-WebRequest -Uri https://tukaani.org/xz/xz-5.2.5-windows.zip -OutFile xz.zip
+Expand-Archive -LiteralPath xz.zip -DestinationPath .
+copy bin_i686\xz.exe . # get it in the path for tar
+
+Invoke-WebRequest -Uri https://github.com/llvm/llvm-project/releases/download/llvmorg-11.0.0/llvm-11.0.0.src.tar.xz -OutFile llvm-11.0.0.src.tar.xz
+
+dir
+
+./xz -d --force llvm-11.0.0.src.tar.xz
+
+tar -xf llvm-11.0.0.src.tar
+
+
+cd llvm-11.0.0.src
+mkdir build
+dir
+cd build
+
+
+# TODO Release build
+& "$env:CMakePath" -G "Visual Studio 16 2019" -DCMAKE_BUILD_TYPE=Debug -DLLVM_USE_CRT_DEBUG=MTd -Thost=x64 ..
+
+& "$env:CMakePath" --build . --target LLVMCore
+& "$env:CMakePath" --build . --target LLVMBitWriter
+#& "$env:CMakePath" --build . --target LLVMDebugInfoDwarf
+
+dir
+
+
+

src/coreclr/inc/clr_std/utility

@@ -49,6 +49,22 @@ namespace std
     {   // forward _Arg, given explicitly specified type parameter
         return ((T&&)_Arg);
     }
+
+    template <class T>
+    constexpr const T&
+    (min)(const T& _Left, const T& _Right)
+    {
+        // return smaller of _Left and _Right
+        return _Right < _Left ? _Right : _Left;
+    }
+
+    template <class T>
+    constexpr const T&
+    (max)(const T& _Left, const T& _Right)
+    {
+        // return larger of _Left and _Right
+        return _Left < _Right ? _Right : _Left;
+    }
 }
 
 namespace std

src/coreclr/inc/cordebuginfo.h

@@ -145,6 +145,8 @@ class ICorDebugInfo
         REGNUM_R13,
         REGNUM_R14,
         REGNUM_R15,
+#elif TARGET_WASM
+// TODO????
 #else
         PORTABILITY_WARNING("Register numbers not defined on this platform")
 #endif

src/coreclr/inc/corjit.h

@@ -145,6 +145,9 @@ enum CheckedWriteBarrierKinds {
 #include "corjithost.h"
 
 extern "C" void jitStartup(ICorJitHost* host);
+#if TARGET_WASM
+extern "C" void jitShutdown(bool processIsTerminating);
+#endif
 
 class ICorJitCompiler;
 class ICorJitInfo;

src/coreclr/inc/crosscomp.h

@@ -26,6 +26,10 @@
 #endif
 #endif // TARGET_WINDOWS
 
+#ifdef TARGET_WASM
+#include "wasm.h"
+#endif
+
 #ifdef UNICODE
 #define MAKE_TARGET_DLLNAME(name) MAKE_TARGET_DLLNAME_W(name)
 #else

src/coreclr/inc/daccess.h

@@ -2371,6 +2371,11 @@ typedef S8PTR(const char)     PTR_CUTF8;
 typedef S16PTR(WCHAR)         PTR_WSTR;
 typedef S16PTR(const WCHAR)   PTR_CWSTR;
 
+#if TARGET_WASM
+#define T_CONTEXT CONTEXT
+#define PT_RUNTIME_FUNCTION PRUNTIME_FUNCTION
+#endif
+
 typedef DPTR(T_CONTEXT)                  PTR_CONTEXT;
 typedef DPTR(PTR_CONTEXT)                PTR_PTR_CONTEXT;
 typedef DPTR(struct _EXCEPTION_POINTERS) PTR_EXCEPTION_POINTERS;

src/coreclr/inc/switches.h

@@ -65,6 +65,8 @@
     #define USE_UPPER_ADDRESS       0
 #endif // !HOST_UNIX
 
+#elif defined(TARGET_WASM)
+    #define USE_UPPER_ADDRESS       0 // not used but is required to be defined
 #else
     #error Please add a new #elif clause and define all portability macros for the new platform
 #endif

src/coreclr/jit/CMakeLists.txt

@@ -3,6 +3,7 @@ set(CMAKE_INCLUDE_CURRENT_DIR ON)
 include_directories("./jitstd")
 include_directories("../inc")
 
+
 if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
   add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-fpermissive>)
   add_compile_options(-Wno-error)
@@ -23,7 +24,11 @@ function(create_standalone_jit)
     endif()
     set(JIT_ARCH_LINK_LIBRARIES gcinfo_unix_arm64)
   else()
-    set(JIT_ARCH_LINK_LIBRARIES gcinfo_${TARGETDETAILS_OS}_${TARGETDETAILS_ARCH})
+    if(TARGETDETAILS_OS STREQUAL "browser")
+      set(JIT_ARCH_LINK_LIBRARIES gcinfo_win_x64) # TODO: Wasm
+    else()
+      set(JIT_ARCH_LINK_LIBRARIES gcinfo_${TARGETDETAILS_OS}_${TARGETDETAILS_ARCH})
+    endif()
   endif()
 
   if(TARGETDETAILS_ARCH STREQUAL "x64")
@@ -34,6 +39,10 @@ function(create_standalone_jit)
     set(JIT_ARCH_SOURCES ${JIT_I386_SOURCES})
   elseif(TARGETDETAILS_ARCH STREQUAL "arm64")
     set(JIT_ARCH_SOURCES ${JIT_ARM64_SOURCES})
+  elseif(TARGETDETAILS_ARCH STREQUAL "wasm64")
+    set(JIT_ARCH_SOURCES ${JIT_WASM64_SOURCES})
+  elseif(TARGETDETAILS_ARCH STREQUAL "wasm32")
+    set(JIT_ARCH_SOURCES ${JIT_WASM32_SOURCES})
   else()
     clr_unknown_arch()
   endif()
@@ -57,6 +66,25 @@ function(create_standalone_jit)
     target_compile_definitions(${TARGETDETAILS_TARGET} PRIVATE FEATURE_SIMD)
     target_compile_definitions(${TARGETDETAILS_TARGET} PRIVATE FEATURE_HW_INTRINSICS)
   endif ()
+
+  if (TARGETDETAILS_ARCH STREQUAL "wasm64" OR TARGETDETAILS_ARCH STREQUAL "wasm32")
+    set(CLR_CMAKE_TARGET_ARCH_WASM 1)
+    target_compile_definitions(${TARGETDETAILS_TARGET} PRIVATE TARGET_WASM)
+    if (TARGETDETAILS_ARCH STREQUAL "wasm32")
+      target_compile_definitions(${TARGETDETAILS_TARGET} PRIVATE TARGET_WASM32)
+    elseif (TARGETDETAILS_ARCH STREQUAL "wasm64")
+      target_compile_definitions(${TARGETDETAILS_TARGET} PRIVATE TARGET_WASM64)
+    endif()
+    target_compile_definitions(${TARGETDETAILS_TARGET} PRIVATE USE_STL)
+    target_compile_definitions(${TARGETDETAILS_TARGET} PRIVATE PAL_STDCPP_COMPAT)
+
+    find_package(LLVM REQUIRED CONFIG PATHS $ENV{LLVM_CMAKE_CONFIG})
+    include_directories(${LLVM_INCLUDE_DIRS})
+    add_definitions(${LLVM_DEFINITIONS})
+    llvm_map_components_to_libnames(llvm_libs core bitwriter)
+    target_link_libraries(${TARGETDETAILS_TARGET} ${llvm_libs})
+  endif ()
+
 endfunction()
 
 if (CLR_CMAKE_TARGET_ARCH_AMD64 OR CLR_CMAKE_TARGET_ARCH_ARM64 OR (CLR_CMAKE_TARGET_ARCH_I386 AND NOT CLR_CMAKE_HOST_UNIX))
@@ -211,6 +239,7 @@ if (CLR_CMAKE_TARGET_WIN32)
     jitstd.h
     jittelemetry.h
     lir.h
+    llvm.h
     loopcloning.h
     loopcloningopts.h
     lower.h
@@ -251,6 +280,7 @@ if (CLR_CMAKE_TARGET_WIN32)
     valuenumtype.h
     varset.h
     vartype.h
+    wasm.h
   )
 
   if (CLR_CMAKE_TARGET_ARCH_ARM64 OR CLR_CMAKE_TARGET_ARCH_ARM)
@@ -293,6 +323,26 @@ set( JIT_AMD64_SOURCES
   hwintrinsiccodegenxarch.cpp
 )
 
+# TODO this is just a copy of AMD64_SOURCES to get started - e.g. simd,hwintrinsics doesn't make sense for wasm
+set( JIT_WASM64_SOURCES
+  simd.cpp
+  simdashwintrinsic.cpp
+  simdcodegenxarch.cpp
+  targetwasm.cpp
+  hwintrinsicxarch.cpp
+  hwintrinsiccodegenxarch.cpp
+  llvm.cpp
+)
+set( JIT_WASM32_SOURCES
+  simd.cpp
+  simdashwintrinsic.cpp
+  simdcodegenxarch.cpp
+  targetwasm.cpp
+  hwintrinsicxarch.cpp
+  hwintrinsiccodegenxarch.cpp
+  llvm.cpp
+)
+
 set( JIT_ARM_SOURCES
   codegenarmarch.cpp
   codegenarm.cpp
@@ -363,6 +413,8 @@ convert_to_absolute_path(JIT_AMD64_SOURCES ${JIT_AMD64_SOURCES})
 convert_to_absolute_path(JIT_ARM_SOURCES ${JIT_ARM_SOURCES})
 convert_to_absolute_path(JIT_I386_SOURCES ${JIT_I386_SOURCES})
 convert_to_absolute_path(JIT_ARM64_SOURCES ${JIT_ARM64_SOURCES})
+convert_to_absolute_path(JIT_WASM64_SOURCES ${JIT_WASM64_SOURCES})
+convert_to_absolute_path(JIT_WASM32_SOURCES ${JIT_WASM32_SOURCES})
 
 set(JIT_DLL_MAIN_FILE ${CMAKE_CURRENT_LIST_DIR}/dllmain.cpp)
 
@@ -493,17 +545,41 @@ if (CLR_CMAKE_BUILD_SUBSET_ALLJITS AND NOT CLR_CROSS_COMPONENTS_BUILD)
   create_standalone_jit(TARGET clrjit_unix_arm_${ARCH_HOST_NAME} OS unix ARCH arm)
   create_standalone_jit(TARGET clrjit_win_arm_${ARCH_HOST_NAME} OS win ARCH arm)
   create_standalone_jit(TARGET clrjit_win_x86_${ARCH_HOST_NAME} OS win ARCH x86)
+  if (NOT CLR_CMAKE_HOST_UNIX AND BUILD_WASM_JIT)
+    # the LLVM clrjit needs to be the last clrjit to use create_standalone_jit as it modifies some cmake variables
+    # LLVM clrjit has an extra export - registerLlvmCallbacks
+    set(CLRJIT_EXPORTS ${CMAKE_CURRENT_LIST_DIR}/ClrJit.Llvm.exports)
+    set(JIT_EXPORTS_FILE ${CMAKE_CURRENT_BINARY_DIR}/ClrJit.Llvm.exports.def)
+    preprocess_file (${CLRJIT_EXPORTS} ${JIT_EXPORTS_FILE})
+    set(JIT_DEF_FILE ${JIT_EXPORTS_FILE})
+
+    # exclude cpp files that are not required when not processing beyond rationalized LIR
+    # use REGEX as this list conatins the absolute paths
+    list(FILTER JIT_CORE_SOURCES EXCLUDE REGEX stacklevelsetter\.cpp)
+    list(FILTER JIT_CORE_SOURCES EXCLUDE REGEX codegencommon\.cpp)
+    list(FILTER JIT_CORE_SOURCES EXCLUDE REGEX codegenlinear\.cpp)
+    list(FILTER JIT_CORE_SOURCES EXCLUDE REGEX emit\.cpp)
+    list(FILTER JIT_CORE_SOURCES EXCLUDE REGEX gcencode\.cpp)
+    list(FILTER JIT_CORE_SOURCES EXCLUDE REGEX instr\.cpp)
+    list(FILTER JIT_CORE_SOURCES EXCLUDE REGEX lower\.cpp)
+    list(FILTER JIT_CORE_SOURCES EXCLUDE REGEX lsra\.cpp)
+    list(FILTER JIT_CORE_SOURCES EXCLUDE REGEX lsrabuild\.cpp)
+    list(FILTER JIT_CORE_SOURCES EXCLUDE REGEX regalloc\.cpp)
+
+    create_standalone_jit(TARGET clrjit_browser_wasm32_${ARCH_HOST_NAME} OS browser ARCH wasm32)
+    # uncomment to enable 8 byte pointer size version of the wasm clrjit.dll
+    #create_standalone_jit(TARGET clrjit_browser_wasm64_${ARCH_HOST_NAME} OS browser ARCH wasm64)
+  endif (NOT CLR_CMAKE_HOST_UNIX AND BUILD_WASM_JIT)
 else()
   if (CLR_CMAKE_TARGET_UNIX)
     create_standalone_jit(TARGET clrjit_unix_${ARCH_TARGET_NAME}_${ARCH_HOST_NAME} OS unix ARCH ${ARCH_TARGET_NAME})
   endif(CLR_CMAKE_TARGET_UNIX)
 endif (CLR_CMAKE_BUILD_SUBSET_ALLJITS AND NOT CLR_CROSS_COMPONENTS_BUILD)
 
-
 if (CLR_CMAKE_TARGET_WIN32 AND CLR_CMAKE_PGO_INSTRUMENT)
   # Copy PGO dependency to target dir
   set(PGORT_DLL "pgort140.dll")
   find_path(PGORT_DIR ${PGORT_DLL} REQUIRED)
   _install(FILES "${PGORT_DIR}/${PGORT_DLL}" DESTINATION ${CMAKE_INSTALL_PREFIX})
   _install(FILES "${PGORT_DIR}/${PGORT_DLL}" DESTINATION ${CMAKE_INSTALL_PREFIX}/sharedFramework)
-endif ()
+endif ()

src/coreclr/jit/ClrJit.Llvm.exports

@@ -0,0 +1,9 @@
+; Licensed to the .NET Foundation under one or more agreements.
+; The .NET Foundation licenses this file to you under the MIT license.
+
+EXPORTS
+    getJit
+    jitStartup
+    jitShutdown
+    registerLlvmCallbacks
+

src/coreclr/jit/codegen.h

@@ -6,7 +6,7 @@
 // of a method, except for the target-specific elements, which are
 // primarily in the Target class.
 //
-
+#ifndef TARGET_WASM
 #ifndef _CODEGEN_H_
 #define _CODEGEN_H_
 #include "codegeninterface.h"
@@ -1608,3 +1608,4 @@ inline void DoPhase(CodeGen* _codeGen, Phases _phase, void (CodeGen::*_action)()
 }
 
 #endif // _CODEGEN_H_
+#endif // TARGET_WASM