Auto merge of #52020 - SimonSapin:no-jemalloc, r=<try>

[Do not merge yet] Remove alloc_jemalloc, switch the default global allocator to System

Fixes #36963

**Do not merge** yet. This PR by itself is likely to regress rustc performance. The purpose of opening it now is to measure by how much. We’ll likely want to figure out #51038 and land them around the same time.

Author: bors

.gitmodules

@@ -2,9 +2,6 @@
 	path = src/llvm
 	url = https://github.com/rust-lang/llvm.git
 	branch = master
-[submodule "src/jemalloc"]
-	path = src/jemalloc
-	url = https://github.com/rust-lang/jemalloc.git
 [submodule "src/rust-installer"]
 	path = src/tools/rust-installer
 	url = https://github.com/rust-lang/rust-installer.git

COPYRIGHT

@@ -229,35 +229,3 @@ their own copyright notices and license terms:
     NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
     USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
     OF SUCH DAMAGE.  */
-
-* jemalloc, under src/jemalloc:
-
-    Copyright (C) 2002-2014 Jason Evans
-    <[email protected]>. All rights reserved.
-    Copyright (C) 2007-2012 Mozilla Foundation.
-    All rights reserved.
-    Copyright (C) 2009-2014 Facebook, Inc.
-    All rights reserved.
-
-    Redistribution and use in source and binary forms, with or without
-    modification, are permitted provided that the following conditions are met:
-    1. Redistributions of source code must retain the above copyright notice(s),
-       this list of conditions and the following disclaimer.
-    2. Redistributions in binary form must reproduce the above copyright notice(s),
-       this list of conditions and the following disclaimer in the documentation
-       and/or other materials provided with the distribution.
-
-    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S)
-    ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
-    INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-    MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-    DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT HOLDER(S)
-    BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
-    EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-    LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-    LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-    HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
-    IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
-    NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
-    USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
-    OF SUCH DAMAGE.

config.toml.example

@@ -270,12 +270,6 @@
 # Adding debuginfo makes them several times larger.
 #debuginfo-tools = false
 
-# Whether or not jemalloc is built and enabled
-#use-jemalloc = true
-
-# Whether or not jemalloc is built with its debug option set
-#debug-jemalloc = false
-
 # Whether or not `panic!`s generate backtraces (RUST_BACKTRACE)
 #backtrace = true
 
@@ -391,10 +385,6 @@
 # target.
 #llvm-config = "../path/to/llvm/root/bin/llvm-config"
 
-# Path to the custom jemalloc static library to link into the standard library
-# by default. This is only used if jemalloc is still enabled above
-#jemalloc = "/path/to/jemalloc/libjemalloc_pic.a"
-
 # If this target is for Android, this option will be required to specify where
 # the NDK for the target lives. This is used to find the C compiler to link and
 # build native code.

src/Cargo.lock

@@ -15,17 +15,6 @@ dependencies = [
  "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "alloc_jemalloc"
-version = "0.0.0"
-dependencies = [
- "build_helper 0.1.0",
- "cc 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)",
- "compiler_builtins 0.0.0",
- "core 0.0.0",
- "libc 0.0.0",
-]
-
 [[package]]
 name = "alloc_system"
 version = "0.0.0"
@@ -2638,7 +2627,6 @@ name = "std"
 version = "0.0.0"
 dependencies = [
  "alloc 0.0.0",
- "alloc_jemalloc 0.0.0",
  "alloc_system 0.0.0",
  "build_helper 0.1.0",
  "cc 1.0.15 (registry+https://github.com/rust-lang/crates.io-index)",

src/bootstrap/bootstrap.py

@@ -696,11 +696,6 @@ def update_submodules(self):
                 backends = self.get_toml('codegen-backends')
                 if backends is None or not 'emscripten' in backends:
                     continue
-            if module.endswith("jemalloc"):
-                if self.get_toml('use-jemalloc') == 'false':
-                    continue
-                if self.get_toml('jemalloc'):
-                    continue
             if module.endswith("lld"):
                 config = self.get_toml('lld')
                 if config is None or config == 'false':

src/bootstrap/compile.rs

@@ -163,16 +163,7 @@ pub fn std_cargo(builder: &Builder,
             .arg("--manifest-path")
             .arg(builder.src.join("src/rustc/compiler_builtins_shim/Cargo.toml"));
     } else {
-        let mut features = builder.std_features();
-
-        // When doing a local rebuild we tell cargo that we're stage1 rather than
-        // stage0. This works fine if the local rust and being-built rust have the
-        // same view of what the default allocator is, but fails otherwise. Since
-        // we don't have a way to express an allocator preference yet, work
-        // around the issue in the case of a local rebuild with jemalloc disabled.
-        if compiler.stage == 0 && builder.local_rebuild && !builder.config.use_jemalloc {
-            features.push_str(" force_alloc_system");
-        }
+        let features = builder.std_features();
 
         if compiler.stage != 0 && builder.config.sanitizers {
             // This variable is used by the sanitizer runtime crates, e.g.
@@ -193,11 +184,6 @@ pub fn std_cargo(builder: &Builder,
             .arg("--manifest-path")
             .arg(builder.src.join("src/libstd/Cargo.toml"));
 
-        if let Some(target) = builder.config.target_config.get(&target) {
-            if let Some(ref jemalloc) = target.jemalloc {
-                cargo.env("JEMALLOC_OVERRIDE", jemalloc);
-            }
-        }
         if target.contains("musl") {
             if let Some(p) = builder.musl_root(target) {
                 cargo.env("MUSL_ROOT", p);

src/bootstrap/config.rs

@@ -117,8 +117,6 @@ pub struct Config {
     pub dist_gpg_password_file: Option<PathBuf>,
 
     // libstd features
-    pub debug_jemalloc: bool,
-    pub use_jemalloc: bool,
     pub backtrace: bool, // support for RUST_BACKTRACE
     pub wasm_syscall: bool,
 
@@ -157,7 +155,6 @@ pub struct Config {
 pub struct Target {
     /// Some(path to llvm-config) if using an external LLVM.
     pub llvm_config: Option<PathBuf>,
-    pub jemalloc: Option<PathBuf>,
     pub cc: Option<PathBuf>,
     pub cxx: Option<PathBuf>,
     pub ar: Option<PathBuf>,
@@ -288,8 +285,6 @@ struct Rust {
     debuginfo_only_std: Option<bool>,
     debuginfo_tools: Option<bool>,
     experimental_parallel_queries: Option<bool>,
-    debug_jemalloc: Option<bool>,
-    use_jemalloc: Option<bool>,
     backtrace: Option<bool>,
     default_linker: Option<String>,
     channel: Option<String>,
@@ -319,7 +314,6 @@ struct Rust {
 #[serde(deny_unknown_fields, rename_all = "kebab-case")]
 struct TomlTarget {
     llvm_config: Option<String>,
-    jemalloc: Option<String>,
     cc: Option<String>,
     cxx: Option<String>,
     ar: Option<String>,
@@ -344,7 +338,6 @@ impl Config {
         config.llvm_enabled = true;
         config.llvm_optimize = true;
         config.llvm_version_check = true;
-        config.use_jemalloc = true;
         config.backtrace = true;
         config.rust_optimize = true;
         config.rust_optimize_tests = true;
@@ -479,7 +472,6 @@ impl Config {
         let mut debuginfo_only_std = None;
         let mut debuginfo_tools = None;
         let mut debug = None;
-        let mut debug_jemalloc = None;
         let mut debuginfo = None;
         let mut debug_assertions = None;
         let mut optimize = None;
@@ -519,12 +511,10 @@ impl Config {
             debuginfo_tools = rust.debuginfo_tools;
             optimize = rust.optimize;
             ignore_git = rust.ignore_git;
-            debug_jemalloc = rust.debug_jemalloc;
             set(&mut config.rust_optimize_tests, rust.optimize_tests);
             set(&mut config.rust_debuginfo_tests, rust.debuginfo_tests);
             set(&mut config.codegen_tests, rust.codegen_tests);
             set(&mut config.rust_rpath, rust.rpath);
-            set(&mut config.use_jemalloc, rust.use_jemalloc);
             set(&mut config.backtrace, rust.backtrace);
             set(&mut config.channel, rust.channel.clone());
             set(&mut config.rust_dist_src, rust.dist_src);
@@ -566,9 +556,6 @@ impl Config {
                 if let Some(ref s) = cfg.llvm_config {
                     target.llvm_config = Some(config.src.join(s));
                 }
-                if let Some(ref s) = cfg.jemalloc {
-                    target.jemalloc = Some(config.src.join(s));
-                }
                 if let Some(ref s) = cfg.android_ndk {
                     target.ndk = Some(config.src.join(s));
                 }
@@ -609,7 +596,6 @@ impl Config {
         config.rust_debuginfo_tools = debuginfo_tools.unwrap_or(false);
 
         let default = debug == Some(true);
-        config.debug_jemalloc = debug_jemalloc.unwrap_or(default);
         config.rust_debuginfo = debuginfo.unwrap_or(default);
         config.rust_debug_assertions = debug_assertions.unwrap_or(default);
         config.rust_optimize = optimize.unwrap_or(!default);

src/bootstrap/configure.py

@@ -80,7 +80,6 @@ def v(*args):
 o("debuginfo-lines", "rust.debuginfo-lines", "build with line number debugger metadata")
 o("debuginfo-only-std", "rust.debuginfo-only-std", "build only libstd with debugging information")
 o("debuginfo-tools", "rust.debuginfo-tools", "build extended tools with debugging information")
-o("debug-jemalloc", "rust.debug-jemalloc", "build jemalloc with --enable-debug --enable-fill")
 v("save-toolstates", "rust.save-toolstates", "save build and test status of external tools into this file")
 
 v("prefix", "install.prefix", "set installation prefix")
@@ -95,7 +94,6 @@ def v(*args):
 
 v("llvm-root", None, "set LLVM root")
 v("python", "build.python", "set path to python")
-v("jemalloc-root", None, "set directory where libjemalloc_pic.a is located")
 v("android-cross-path", "target.arm-linux-androideabi.android-ndk",
   "Android NDK standalone path (deprecated)")
 v("i686-linux-android-ndk", "target.i686-linux-android.android-ndk",
@@ -144,7 +142,6 @@ def v(*args):
 # Many of these are saved below during the "writing configuration" step
 # (others are conditionally saved).
 o("manage-submodules", "build.submodules", "let the build manage the git submodules")
-o("jemalloc", "rust.use-jemalloc", "build liballoc with jemalloc")
 o("full-bootstrap", "build.full-bootstrap", "build three compilers instead of two")
 o("extended", "build.extended", "build an extended rust tool set")
 
@@ -322,8 +319,6 @@ def set(key, value):
         set('build.cargo', value + '/bin/cargo')
     elif option.name == 'llvm-root':
         set('target.{}.llvm-config'.format(build()), value + '/bin/llvm-config')
-    elif option.name == 'jemalloc-root':
-        set('target.{}.jemalloc'.format(build()), value + '/libjemalloc_pic.a')
     elif option.name == 'tools':
         set('build.tools', value.split(','))
     elif option.name == 'host':

src/bootstrap/dist.rs

@@ -834,7 +834,6 @@ impl Step for Src {
             "src/build_helper",
             "src/dlmalloc",
             "src/liballoc",
-            "src/liballoc_jemalloc",
             "src/liballoc_system",
             "src/libbacktrace",
             "src/libcompiler_builtins",
@@ -854,13 +853,11 @@ impl Step for Src {
             "src/rustc/dlmalloc_shim",
             "src/libtest",
             "src/libterm",
-            "src/jemalloc",
             "src/libprofiler_builtins",
             "src/stdsimd",
         ];
         let std_src_dirs_exclude = [
             "src/libcompiler_builtins/compiler-rt/test",
-            "src/jemalloc/test/unit",
         ];
 
         copy_src_dirs(builder, &std_src_dirs[..], &std_src_dirs_exclude[..], &dst_src);

src/bootstrap/lib.rs

@@ -498,12 +498,6 @@ impl Build {
     fn std_features(&self) -> String {
         let mut features = "panic-unwind".to_string();
 
-        if self.config.debug_jemalloc {
-            features.push_str(" debug-jemalloc");
-        }
-        if self.config.use_jemalloc {
-            features.push_str(" jemalloc");
-        }
         if self.config.backtrace {
             features.push_str(" backtrace");
         }
@@ -518,11 +512,7 @@ impl Build {
 
     /// Get the space-separated set of activated features for the compiler.
     fn rustc_features(&self) -> String {
-        let mut features = String::new();
-        if self.config.use_jemalloc {
-            features.push_str(" jemalloc");
-        }
-        features
+        String::new()
     }
 
     /// Component directory that Cargo will produce output into (e.g.
@@ -732,7 +722,7 @@ impl Build {
         // If we're compiling on macOS then we add a few unconditional flags
         // indicating that we want libc++ (more filled out than libstdc++) and
         // we want to compile for 10.7. This way we can ensure that
-        // LLVM/jemalloc/etc are all properly compiled.
+        // LLVM/etc are all properly compiled.
         if target.contains("apple-darwin") {
             base.push("-stdlib=libc++".into());
         }

src/bootstrap/sanity.rs

@@ -152,12 +152,6 @@ pub fn check(build: &mut Build) {
         if !build.config.dry_run {
             cmd_finder.must_have(build.cxx(*host).unwrap());
         }
-
-        // The msvc hosts don't use jemalloc, turn it off globally to
-        // avoid packaging the dummy liballoc_jemalloc on that platform.
-        if host.contains("msvc") {
-            build.config.use_jemalloc = false;
-        }
     }
 
     // Externally configured LLVM requires FileCheck to exist

src/bootstrap/test.rs

@@ -1466,8 +1466,7 @@ impl Step for CrateNotDefault {
     type Output = ();
 
     fn should_run(run: ShouldRun) -> ShouldRun {
-        run.path("src/liballoc_jemalloc")
-            .path("src/librustc_asan")
+        run.path("src/librustc_asan")
             .path("src/librustc_lsan")
             .path("src/librustc_msan")
             .path("src/librustc_tsan")
@@ -1484,7 +1483,6 @@ impl Step for CrateNotDefault {
             target: run.target,
             test_kind,
             krate: match run.path {
-                _ if run.path.ends_with("src/liballoc_jemalloc") => "alloc_jemalloc",
                 _ if run.path.ends_with("src/librustc_asan") => "rustc_asan",
                 _ if run.path.ends_with("src/librustc_lsan") => "rustc_lsan",
                 _ if run.path.ends_with("src/librustc_msan") => "rustc_msan",
@@ -1523,7 +1521,6 @@ impl Step for Crate {
         run = run.krate("test");
         for krate in run.builder.in_tree_crates("std") {
             if krate.is_local(&run.builder)
-                && !krate.name.contains("jemalloc")
                 && !(krate.name.starts_with("rustc_") && krate.name.ends_with("san"))
                 && krate.name != "dlmalloc"
             {

src/ci/docker/disabled/dist-x86_64-haiku/Dockerfile

@@ -42,7 +42,7 @@ RUN sh /scripts/sccache.sh
 ENV HOST=x86_64-unknown-haiku
 ENV TARGET=target.$HOST
 
-ENV RUST_CONFIGURE_ARGS --disable-jemalloc \
+ENV RUST_CONFIGURE_ARGS \
   --set=$TARGET.cc=x86_64-unknown-haiku-gcc \
   --set=$TARGET.cxx=x86_64-unknown-haiku-g++ \
   --set=$TARGET.llvm-config=/bin/llvm-config-haiku

src/jemalloc

@@ -14,7 +14,7 @@ use std::alloc::{Global, Alloc, Layout};
 /// https://github.com/rust-lang/rust/issues/45955
 ///
 /// Note that `#[global_allocator]` is not used,
-/// so `liballoc_jemalloc` is linked (on some platforms).
+/// so `System` and `Global` might be backed by different allocators.
 #[test]
 fn alloc_system_overaligned_request() {
     check_overalign_requests(System)