Compile rustc for wasm15 with llvm

oligamiq · bjorn3 · commit d77d78b18c06 · 2024-09-23T19:11:01.000+02:00
diff --git a/.gitmodules b/.gitmodules
@@ -32,8 +32,8 @@
 	shallow = true
 [submodule "src/llvm-project"]
 	path = src/llvm-project
-	url = https://github.com/rust-lang/llvm-project.git
-	branch = rustc/19.1-2024-07-30
+	url = https://github.com/YoWASP/llvm-project
+	branch = main+wasm
 	shallow = true
 [submodule "src/doc/embedded-book"]
 	path = src/doc/embedded-book
diff --git a/Cargo.lock b/Cargo.lock
@@ -407,9 +407,12 @@ version = "0.1.0"
 
 [[package]]
 name = "cc"
-version = "1.0.105"
+version = "1.1.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5208975e568d83b6b05cc0a063c8e7e9acc2b43bee6da15616a5b73e109d7437"
+checksum = "57b6a275aa2903740dc87da01c62040406b8812552e97129a63ea8850a17c6e6"
+dependencies = [
+ "shlex",
+]
 
 [[package]]
 name = "cfg-if"
diff --git a/comment.txt b/comment.txt
@@ -16,3 +16,25 @@ $ gcc -fuse-ld=lld tmp/rmeta*/lib.rmeta tmp/rust_out.* dist/lib/rustlib/x86_64-u
 $ ./rust_out
 Hello World!
 ```
+
+use LLVM
+Install example:
+WASI_SDK_PATH=`pwd`/wasi-sdk-24.0-x86_64-linux WASI_SYSROOT=`pwd`/wasi-sdk-24.0-x86_64-linux/share/wasi-sysroot ./x.py install
+
+If you just want to run it, https://github.com/oligamiq/rust_wasm/tree/main/rustc_llvm
+```
+$ mkdir tmp
+$ echo 'fn main() { println!("Hello World!"); }' | wasmtime run -Sthreads=y -Spreview2=n --dir tmp::/ --dir dist --env RUST_MIN_STACK=16777216 dist/bin/rustc.wasm - --sysroot dist --target wasm32-wasip1-threads -Csave-temps
+$ gcc -fuse-ld=lld tmp/rmeta*/lib.rmeta tmp/rust_out.*.o dist/lib/rustlib/x86_64-unknown-linux-gnu/lib/lib*.rlib -o rust_out
+$ ./rust_out
+Hello World!
+```
+
+to Wasi
+```
+$ mkdir tmp
+$ echo 'fn main() { println!("Hello World!"); }' | wasmtime run -Sthreads=y -Spreview2=n --dir tmp::/ --dir dist --env RUST_MIN_STACK=16777216 dist/bin/rustc.wasm - --sysroot dist --target wasm32-wasip1-threads -Csave-temps
+$ wasi-sdk-24.0-x86_64-linux/bin/wasm-ld --shared-memory --max-memory=1073741824 --import-memory --export __main_void -z stack-size=1048576 --stack-first --allow-undefined --no-demangle --import-memory --export-memory --shared-memory dist/lib/rustlib/wasm32-wasip1-threads/lib/self-contained/crt1-command.o tmp/rust_out.*.o dist/lib/rustlib/wasm32-wasip1-threads/lib/lib*.rlib -L dist/lib/rustlib/wasm32-wasip1-threads/lib/self-contained -lc -o rust_out.wasm
+$ wasmtime run -Sthreads=y rust_out.wasm
+Hello World!
+```
diff --git a/compiler/rustc_fs_util/src/lib.rs b/compiler/rustc_fs_util/src/lib.rs
@@ -84,7 +84,7 @@ pub fn path_to_c_string(p: &Path) -> CString {
     let p: &OsStr = p.as_ref();
     CString::new(p.as_bytes()).unwrap()
 }
-#[cfg(windows)]
+#[cfg(any(windows, target_os = "wasi"))]
 pub fn path_to_c_string(p: &Path) -> CString {
     CString::new(p.to_str().unwrap()).unwrap()
 }
diff --git a/compiler/rustc_llvm/Cargo.toml b/compiler/rustc_llvm/Cargo.toml
@@ -10,5 +10,5 @@ libc = "0.2.73"
 
 [build-dependencies]
 # tidy-alphabetical-start
-cc = "=1.0.105" # FIXME(cc): pinned to keep support for VS2013
+cc = "1.1.15"
 # tidy-alphabetical-end
diff --git a/compiler/rustc_llvm/build.rs b/compiler/rustc_llvm/build.rs
@@ -102,6 +102,10 @@ fn output(cmd: &mut Command) -> String {
 }
 
 fn main() {
+    if env::var("TARGET").expect("TARGET was not set").contains("wasi") {
+        std::env::var("WASI_SYSROOT").expect("WASI_SYSROOT not set");
+    }
+
     for component in REQUIRED_COMPONENTS.iter().chain(OPTIONAL_COMPONENTS.iter()) {
         println!("cargo:rustc-check-cfg=cfg(llvm_component,values(\"{component}\"))");
     }
@@ -201,6 +205,16 @@ fn main() {
         cfg.define("NDEBUG", None);
     }
 
+
+    if target.contains("wasi") {
+        // ref src/bootstrap/src/core/build_steps/llvm.rs
+
+        let wasi_sysroot = env::var("WASI_SYSROOT").expect("WASI_SYSROOT not set");
+        cfg.compiler(format!("{wasi_sysroot}/../../bin/{target}-clang++"));
+        cfg.flag("-pthread");
+        cfg.flag("-D_WASI_EMULATED_MMAN");
+    }
+
     rerun_if_changed_anything_in_dir(Path::new("llvm-wrapper"));
     cfg.file("llvm-wrapper/PassWrapper.cpp")
         .file("llvm-wrapper/RustWrapper.cpp")
diff --git a/compiler/rustc_target/src/spec/targets/wasm32_wasip1_threads.rs b/compiler/rustc_target/src/spec/targets/wasm32_wasip1_threads.rs
@@ -17,7 +17,7 @@ pub(crate) fn target() -> Target {
 
     options.add_pre_link_args(
         LinkerFlavor::WasmLld(Cc::No),
-        &["--import-memory", "--export-memory", "--shared-memory", "-Wl,--max-memory=1073741824"],
+        &["--import-memory", "--export-memory", "--shared-memory", "--max-memory=1073741824", "-lwasi-emulated-mman"],
     );
     options.add_pre_link_args(
         LinkerFlavor::WasmLld(Cc::Yes),
@@ -27,6 +27,7 @@ pub(crate) fn target() -> Target {
             "-Wl,--export-memory,",
             "-Wl,--shared-memory",
             "-Wl,--max-memory=1073741824",
+            "-lwasi-emulated-mman",
         ],
     );
 
diff --git a/config.llvm.toml b/config.llvm.toml
@@ -0,0 +1,40 @@
+# Includes one of the default files in src/bootstrap/defaults
+profile = "compiler"
+change-id = 9999999
+
+[rust]
+codegen-backends = ["llvm"]
+deny-warnings = false
+llvm-bitcode-linker = false
+
+[llvm]
+cflags = "-march=native"
+cxxflags = "-march=native"
+static-libstdcpp = true
+ninja = false
+download-ci-llvm = false
+link-shared = false
+# thin-lto = true
+
+[build]
+docs = false
+extended = false
+tools = []
+# host = ["wasm32-wasip1-threads", "x86_64-unknown-linux-gnu"]
+host = ["wasm32-wasip1-threads"]
+target = ["x86_64-unknown-linux-gnu", "wasm32-wasip1-threads"]
+# target = ["wasm32-wasip1-threads"]
+cargo-native-static = true
+
+[install]
+prefix = "dist"
+sysconfdir = "etc"
+
+[target.'wasm32-wasip1-threads']
+wasi-root = "wasi-sdk-24.0-x86_64-linux/share/wasi-sysroot"
+# codegen-backends = ["cranelift"]
+linker = "wasi-sdk-24.0-x86_64-linux/bin/clang"
+codegen-backends = ["llvm"]
+
+[target.'x86_64-unknown-linux-gnu']
+cc = "gcc"
diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs
@@ -515,6 +515,178 @@ impl Step for Llvm {
             return res;
         }
 
+        if target.contains("wasi") {
+            let wasi_sysroot = env::var("WASI_SYSROOT").expect("WASI_SYSROOT not set");
+            let wasi_sdk_path = std::path::Path::new(&wasi_sysroot).join("../../").canonicalize().expect("invalid WASI_SYSROOT");
+            let wasi_sysroot = format!("--sysroot={wasi_sysroot}");
+            let wasi_sdk_path = wasi_sdk_path.to_str().expect("invalid WASI_SYSROOT");
+            let wasi_target = target.triple.to_string();
+            let wasi_cflags = String::from("");
+            let wasi_ldflags = String::from("");
+            let wasi_target_llvm = target.triple.to_string();
+            let wasi_cflags_llvm = format!("{wasi_cflags} -pthread");
+            let wasi_ldflags_llvm = wasi_ldflags;
+            // LLVM has some (unreachable in our configuration) calls to mmap.
+            let wasi_cflags_llvm = format!("{wasi_cflags_llvm} -D_WASI_EMULATED_MMAN");
+            let wasi_ldflags_llvm = format!("{wasi_ldflags_llvm} -lwasi-emulated-mman");
+            // Depending on the code being compiled, both Clang and LLD can consume unbounded amounts of memory.
+            let wasi_ldflags_llvm = format!("{wasi_ldflags_llvm} -Wl,--max-memory=4294967296");
+            // Compiling C++ code requires a lot of stack space and can overflow and corrupt the heap.
+            // (For example, `#include <iostream>` alone does it in a build with the default stack size.)
+            let wasi_ldflags_llvm = format!("{wasi_ldflags_llvm} -Wl,-z,stack-size=1048576 -Wl,--stack-first");
+            // Some of the host APIs that are statically required by LLVM (notably threading) are dynamically
+            // never used. An LTO build removes imports of these APIs, simplifying deployment
+            let wasi_cflags_llvm = format!("{wasi_cflags_llvm} -flto");
+            let wasi_ldflags_llvm = format!("{wasi_ldflags_llvm} -flto -Wl,--strip-all");
+
+            // We need two toolchain files: one for the compiler itself (which needs threads at the moment since
+            // -DLLVM_ENABLE_THREADS=OFF is kind of broken), and one for the runtime libs.
+            cfg.define("WASI", "TRUE")
+            .define("CMAKE_SYSTEM_NAME", "Generic")
+            .define("CMAKE_SYSTEM_VERSION", "1")
+            .define("CMAKE_SYSTEM_PROCESSOR", "wasm32")
+            .define("CMAKE_EXECUTABLE_SUFFIX", ".wasm")
+            .define("CMAKE_FIND_ROOT_PATH_MODE_PROGRAM", "NEVER")
+            .define("CMAKE_FIND_ROOT_PATH_MODE_LIBRARY", "ONLY")
+            .define("CMAKE_FIND_ROOT_PATH_MODE_INCLUDE", "ONLY")
+            .define("CMAKE_FIND_ROOT_PATH_MODE_PACKAGE", "ONLY")
+            .define("CMAKE_C_COMPILER", format!("{wasi_sdk_path}/bin/{wasi_target}-clang"))
+            .define("CMAKE_CXX_COMPILER", format!("{wasi_sdk_path}/bin/{wasi_target}-clang++"))
+            .define("CMAKE_LINKER", format!("{wasi_sdk_path}/bin/wasm-ld"))
+            .define("CMAKE_AR", format!("{wasi_sdk_path}/bin/ar"))
+            .define("CMAKE_RANLIB", format!("{wasi_sdk_path}/bin/ranlib"))
+            .define("CMAKE_C_COMPILER_TARGET", &wasi_target_llvm)
+            .define("CMAKE_CXX_COMPILER_TARGET", &wasi_target_llvm)
+            .define("CMAKE_C_FLAGS", format!("{wasi_sysroot} {wasi_cflags_llvm}"))
+            .define("CMAKE_CXX_FLAGS", format!("{wasi_sysroot} {wasi_cflags_llvm}"))
+            .define("CMAKE_EXE_LINKER_FLAGS", wasi_ldflags_llvm)
+            .define("LLVM_BUILD_SHARED_LIBS", "OFF")
+            .define("LLVM_ENABLE_PIC", "OFF")
+            .define("LLVM_BUILD_STATIC", "ON")
+            // .define("LLVM_ENABLE_THREADS", "OFF")
+            .define("LLVM_ENABLE_THREADS", "ON")
+            .define("LLVM_BUILD_RUNTIME", "OFF")
+            .define("LLVM_BUILD_TOOLS", "OFF")
+            .define("LLVM_INCLUDE_UTILS", "OFF")
+            .define("LLVM_BUILD_UTILS", "OFF")
+            .define("LLVM_INCLUDE_RUNTIMES", "OFF")
+            .define("LLVM_INCLUDE_EXAMPLES", "OFF")
+            .define("LLVM_INCLUDE_TESTS", "OFF")
+            .define("LLVM_INCLUDE_BENCHMARKS", "OFF")
+            .define("LLVM_INCLUDE_DOCS", "OFF")
+            .define("LLVM_TOOL_BUGPOINT_BUILD", "OFF")
+            .define("LLVM_TOOL_BUGPOINT_PASSES_BUILD", "OFF")
+            .define("LLVM_TOOL_DSYMUTIL_BUILD", "OFF")
+            .define("LLVM_TOOL_DXIL_DIS_BUILD", "OFF")
+            .define("LLVM_TOOL_GOLD_BUILD", "OFF")
+            .define("LLVM_TOOL_LLC_BUILD", "OFF")
+            .define("LLVM_TOOL_LLI_BUILD", "OFF")
+            // .define("LLVM_TOOL_LLVM_AR_BUILD", "ON")
+            // .define("LLVM_TOOL_LLVM_AS_BUILD", "ON")
+            .define("LLVM_TOOL_LLVM_AS_FUZZER_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_BCANALYZER_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_CAT_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_CFI_VERIFY_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_CONFIG_BUILD", "OFF")
+            // .define("LLVM_TOOL_LLVM_COV_BUILD", "ON")
+            .define("LLVM_TOOL_LLVM_CVTRES_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_CXXDUMP_BUILD", "OFF")
+            // .define("LLVM_TOOL_LLVM_CXXFILT_BUILD", "ON")
+            .define("LLVM_TOOL_LLVM_CXXMAP_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_C_TEST_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_DEBUGINFOD_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_DEBUGINFOD_FIND_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_DEBUGINFO_ANALYZER_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_DIFF_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_DIS_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_DIS_FUZZER_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_DLANG_DEMANGLE_FUZZER_BUILD", "OFF")
+            // .define("LLVM_TOOL_LLVM_DRIVER_BUILD", "ON")
+            // .define("LLVM_TOOL_LLVM_DWARFDUMP_BUILD", "ON")
+            .define("LLVM_TOOL_LLVM_DWARFUTIL_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_DWP_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_EXEGESIS_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_EXTRACT_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_GSYMUTIL_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_IFS_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_ISEL_FUZZER_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_ITANIUM_DEMANGLE_FUZZER_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_JITLINK_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_JITLISTENER_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_LIBTOOL_DARWIN_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_LINK_BUILD", "ON")
+            .define("LLVM_TOOL_LLVM_LIPO_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_LTO2_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_LTO_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_MCA_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_MC_ASSEMBLE_FUZZER_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_MC_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_MC_DISASSEMBLE_FUZZER_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_MICROSOFT_DEMANGLE_FUZZER_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_ML_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_MODEXTRACT_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_MT_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_NM_BUILD", "OFF")
+            // .define("LLVM_TOOL_LLVM_OBJCOPY_BUILD", "ON")
+            // .define("LLVM_TOOL_LLVM_OBJDUMP_BUILD", "ON")
+            .define("LLVM_TOOL_LLVM_OPT_FUZZER_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_OPT_REPORT_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_PDBUTIL_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_PROFDATA_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_PROFGEN_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_RC_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_READOBJ_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_READTAPI_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_REDUCE_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_REMARKUTIL_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_RTDYLD_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_RUST_DEMANGLE_FUZZER_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_SHLIB_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_SIM_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_SIZE_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_SPECIAL_CASE_LIST_FUZZER_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_SPLIT_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_STRESS_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_STRINGS_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_SYMBOLIZER_BUILD", "ON")
+            .define("LLVM_TOOL_LLVM_TLI_CHECKER_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_UNDNAME_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_XRAY_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_YAML_NUMERIC_PARSER_FUZZER_BUILD", "OFF")
+            .define("LLVM_TOOL_LLVM_YAML_PARSER_FUZZER_BUILD", "OFF")
+            .define("LLVM_TOOL_LTO_BUILD", "OFF")
+            .define("LLVM_TOOL_OBJ2YAML_BUILD", "OFF")
+            .define("LLVM_TOOL_OPT_BUILD", "OFF")
+            .define("LLVM_TOOL_OPT_VIEWER_BUILD", "OFF")
+            .define("LLVM_TOOL_REDUCE_CHUNK_LIST_BUILD", "OFF")
+            .define("LLVM_TOOL_REMARKS_SHLIB_BUILD", "OFF")
+            .define("LLVM_TOOL_SANCOV_BUILD", "OFF")
+            .define("LLVM_TOOL_SANSTATS_BUILD", "OFF")
+            .define("LLVM_TOOL_SPIRV_TOOLS_BUILD", "OFF")
+            .define("LLVM_TOOL_VERIFY_USELISTORDER_BUILD", "OFF")
+            .define("LLVM_TOOL_VFABI_DEMANGLE_FUZZER_BUILD", "OFF")
+            .define("LLVM_TOOL_XCODE_TOOLCHAIN_BUILD", "OFF")
+            .define("LLVM_TOOL_YAML2OBJ_BUILD", "OFF")
+            // .define("LLVM_ENABLE_PROJECTS", "clang;lld")
+            .define("LLVM_ENABLE_PROJECTS", "")
+            // .define("CLANG_ENABLE_ARCMT", "OFF")
+            // .define("CLANG_ENABLE_STATIC_ANALYZER", "OFF")
+            // .define("CLANG_INCLUDE_TESTS", "OFF")
+            // .define("CLANG_BUILD_TOOLS", "OFF")
+            // .define("CLANG_TOOL_CLANG_SCAN_DEPS_BUILD", "OFF")
+            // .define("CLANG_TOOL_CLANG_INSTALLAPI_BUILD", "OFF")
+            // .define("CLANG_BUILD_EXAMPLES", "OFF")
+            // .define("CLANG_INCLUDE_DOCS", "OFF")
+            // .define("CLANG_LINKS_TO_CREATE", "clang;clang++")
+            // .define("CLANG_LINKS_TO_CREATE", "")
+            .define("LLD_BUILD_TOOLS", "OFF")
+            .define("CMAKE_BUILD_TYPE", "MinSizeRel")
+            .define("HAVE_DLOPEN", "");
+        } else {
+            cfg.define("LLVM_TOOL_LLVM_CONFIG_BUILD", "ON")
+                .define("LLVM_BUILD_TOOLS", "ON");
+        }
+
         cfg.build();
 
         // Helper to find the name of LLVM's shared library on darwin and linux.

Original file line number	Diff line number	Diff line change
`@@ -84,7 +84,7 @@ pub fn path_to_c_string(p: &Path) -> CString {`
`84`	`84`	`let p: &OsStr = p.as_ref();`
`85`	`85`	`CString::new(p.as_bytes()).unwrap()`
`86`	`86`	`}`
`87`		`-#[cfg(windows)]`
	`87`	`+#[cfg(any(windows, target_os = "wasi"))]`
`88`	`88`	`pub fn path_to_c_string(p: &Path) -> CString {`
`89`	`89`	`CString::new(p.to_str().unwrap()).unwrap()`
`90`	`90`	`}`