diff --git a/Cargo.lock b/Cargo.lock
index bfc2d3e066a89..83ac8396a16e3 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -243,6 +243,7 @@ dependencies = [
"anyhow",
"flate2",
"hex 0.4.2",
+ "num_cpus",
"rayon",
"serde",
"serde_json",
diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs
index 4bc162abee6c1..707c1ff3efad9 100644
--- a/src/bootstrap/builder.rs
+++ b/src/bootstrap/builder.rs
@@ -193,37 +193,37 @@ impl StepDescription {
);
}
- if paths.is_empty() {
- for (desc, should_run) in v.iter().zip(should_runs) {
+ if paths.is_empty() || builder.config.include_default_paths {
+ for (desc, should_run) in v.iter().zip(&should_runs) {
if desc.default && should_run.is_really_default {
for pathset in &should_run.paths {
desc.maybe_run(builder, pathset);
}
}
}
- } else {
- for path in paths {
- // strip CurDir prefix if present
- let path = match path.strip_prefix(".") {
- Ok(p) => p,
- Err(_) => path,
- };
+ }
- let mut attempted_run = false;
- for (desc, should_run) in v.iter().zip(&should_runs) {
- if let Some(suite) = should_run.is_suite_path(path) {
- attempted_run = true;
- desc.maybe_run(builder, suite);
- } else if let Some(pathset) = should_run.pathset_for_path(path) {
- attempted_run = true;
- desc.maybe_run(builder, pathset);
- }
- }
+ for path in paths {
+ // strip CurDir prefix if present
+ let path = match path.strip_prefix(".") {
+ Ok(p) => p,
+ Err(_) => path,
+ };
- if !attempted_run {
- panic!("error: no rules matched {}", path.display());
+ let mut attempted_run = false;
+ for (desc, should_run) in v.iter().zip(&should_runs) {
+ if let Some(suite) = should_run.is_suite_path(path) {
+ attempted_run = true;
+ desc.maybe_run(builder, suite);
+ } else if let Some(pathset) = should_run.pathset_for_path(path) {
+ attempted_run = true;
+ desc.maybe_run(builder, pathset);
}
}
+
+ if !attempted_run {
+ panic!("error: no rules matched {}", path.display());
+ }
}
}
}
@@ -462,6 +462,7 @@ impl<'a> Builder<'a> {
dist::LlvmTools,
dist::RustDev,
dist::Extended,
+ dist::BuildManifest,
dist::HashSign
),
Kind::Install => describe!(
diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs
index 6265bbaf5c22c..db82155bd6ad2 100644
--- a/src/bootstrap/config.rs
+++ b/src/bootstrap/config.rs
@@ -61,6 +61,7 @@ pub struct Config {
pub profiler: bool,
pub ignore_git: bool,
pub exclude: Vec<PathBuf>,
+ pub include_default_paths: bool,
pub rustc_error_format: Option<String>,
pub json_output: bool,
pub test_compare_mode: bool,
@@ -532,6 +533,7 @@ impl Config {
let mut config = Config::default_opts();
config.exclude = flags.exclude;
+ config.include_default_paths = flags.include_default_paths;
config.rustc_error_format = flags.rustc_error_format;
config.json_output = flags.json_output;
config.on_fail = flags.on_fail;
diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs
index 3a0743da7a415..dd4cf9d595323 100644
--- a/src/bootstrap/dist.rs
+++ b/src/bootstrap/dist.rs
@@ -2353,7 +2353,6 @@ impl Step for HashSign {
cmd.arg(today.trim());
cmd.arg(addr);
cmd.arg(&builder.config.channel);
- cmd.arg(&builder.src);
cmd.env("BUILD_MANIFEST_LEGACY", "1");
builder.create_dir(&distdir(builder));
@@ -2584,3 +2583,70 @@ impl Step for RustDev {
Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple)))
}
}
+
+/// Tarball containing a prebuilt version of the build-manifest tool, intented to be used by the
+/// release process to avoid cloning the monorepo and building stuff.
+///
+/// Should not be considered stable by end users.
+#[derive(Clone, Debug, Eq, Hash, PartialEq)]
+pub struct BuildManifest {
+ pub target: TargetSelection,
+}
+
+impl Step for BuildManifest {
+ type Output = PathBuf;
+ const DEFAULT: bool = false;
+ const ONLY_HOSTS: bool = true;
+
+ fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+ run.path("src/tools/build-manifest")
+ }
+
+ fn make_run(run: RunConfig<'_>) {
+ run.builder.ensure(BuildManifest { target: run.target });
+ }
+
+ fn run(self, builder: &Builder<'_>) -> PathBuf {
+ let build_manifest = builder.tool_exe(Tool::BuildManifest);
+
+ let name = pkgname(builder, "build-manifest");
+ let tmp = tmpdir(builder);
+
+ // Prepare the image.
+ let image = tmp.join("build-manifest-image");
+ let image_bin = image.join("bin");
+ let _ = fs::remove_dir_all(&image);
+ t!(fs::create_dir_all(&image_bin));
+ builder.install(&build_manifest, &image_bin.join("build-manifest"), 0o755);
+
+ // Prepare the overlay.
+ let overlay = tmp.join("build-manifest-overlay");
+ let _ = fs::remove_dir_all(&overlay);
+ builder.create_dir(&overlay);
+ builder.create(&overlay.join("version"), &builder.rust_version());
+ for file in &["COPYRIGHT", "LICENSE-APACHE", "LICENSE-MIT", "README.md"] {
+ builder.install(&builder.src.join(file), &overlay, 0o644);
+ }
+
+ // Create the final tarball.
+ let mut cmd = rust_installer(builder);
+ cmd.arg("generate")
+ .arg("--product-name=Rust")
+ .arg("--rel-manifest-dir=rustlib")
+ .arg("--success-message=build-manifest installed.")
+ .arg("--image-dir")
+ .arg(&image)
+ .arg("--work-dir")
+ .arg(&tmpdir(builder))
+ .arg("--output-dir")
+ .arg(&distdir(builder))
+ .arg("--non-installed-overlay")
+ .arg(&overlay)
+ .arg(format!("--package-name={}-{}", name, self.target.triple))
+ .arg("--legacy-manifest-dirs=rustlib,cargo")
+ .arg("--component-name=build-manifest");
+
+ builder.run(&mut cmd);
+ distdir(builder).join(format!("{}-{}.tar.gz", name, self.target.triple))
+ }
+}
diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs
index 319a0b4e611eb..c10188875fbc4 100644
--- a/src/bootstrap/flags.rs
+++ b/src/bootstrap/flags.rs
@@ -30,6 +30,7 @@ pub struct Flags {
pub cmd: Subcommand,
pub incremental: bool,
pub exclude: Vec<PathBuf>,
+ pub include_default_paths: bool,
pub rustc_error_format: Option<String>,
pub json_output: bool,
pub dry_run: bool,
@@ -137,6 +138,11 @@ To learn more about a subcommand, run `./x.py <subcommand> -h`",
opts.optmulti("", "host", "host targets to build", "HOST");
opts.optmulti("", "target", "target targets to build", "TARGET");
opts.optmulti("", "exclude", "build paths to exclude", "PATH");
+ opts.optflag(
+ "",
+ "include-default-paths",
+ "include default paths in addition to the provided ones",
+ );
opts.optopt("", "on-fail", "command to run on failure", "CMD");
opts.optflag("", "dry-run", "dry run; don't build anything");
opts.optopt(
@@ -618,6 +624,7 @@ Arguments:
.into_iter()
.map(|p| p.into())
.collect::<Vec<_>>(),
+ include_default_paths: matches.opt_present("include-default-paths"),
deny_warnings: parse_deny_warnings(&matches),
llvm_skip_rebuild: matches.opt_str("llvm-skip-rebuild").map(|s| s.to_lowercase()).map(
|s| s.parse::<bool>().expect("`llvm-skip-rebuild` should be either true or false"),
diff --git a/src/bootstrap/run.rs b/src/bootstrap/run.rs
index 80c093e713eff..7c64e5a0aadc8 100644
--- a/src/bootstrap/run.rs
+++ b/src/bootstrap/run.rs
@@ -77,7 +77,6 @@ impl Step for BuildManifest {
cmd.arg(today.trim());
cmd.arg(addr);
cmd.arg(&builder.config.channel);
- cmd.arg(&builder.src);
builder.create_dir(&distdir(builder));
builder.run(&mut cmd);
diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile
index 58e2567a58f08..14700aeea05af 100644
--- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile
+++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile
@@ -98,7 +98,9 @@ ENV RUST_CONFIGURE_ARGS \
--set llvm.thin-lto=true \
--set llvm.ninja=false \
--set rust.jemalloc
-ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS
+ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS \
+ --include-default-paths \
+ src/tools/build-manifest
ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=clang
# This is the only builder which will create source tarballs
diff --git a/src/tools/build-manifest/Cargo.toml b/src/tools/build-manifest/Cargo.toml
index 4ae4dbfc06ede..4a2c710811f61 100644
--- a/src/tools/build-manifest/Cargo.toml
+++ b/src/tools/build-manifest/Cargo.toml
@@ -14,3 +14,4 @@ tar = "0.4.29"
sha2 = "0.9.1"
rayon = "1.3.1"
hex = "0.4.2"
+num_cpus = "1.13.0"
diff --git a/src/tools/build-manifest/README.md b/src/tools/build-manifest/README.md
index 26e96c9fd8fda..b77c5a907c118 100644
--- a/src/tools/build-manifest/README.md
+++ b/src/tools/build-manifest/README.md
@@ -21,8 +21,8 @@ Then, you can generate the manifest and all the packages from `path/to/dist` to
```
$ cargo +nightly run path/to/dist path/to/output 1970-01-01 http://example.com \
- CHANNEL path/to/rust/repo
+ CHANNEL VERSION
```
Remember to replace `CHANNEL` with the channel you produced dist artifacts of
-and `path/to/rust/repo` with the path to your checkout of the Rust repository.
+and `VERSION` with the current Rust version.
diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs
index 7ee28cd6e5d95..cb04900c737e0 100644
--- a/src/tools/build-manifest/src/main.rs
+++ b/src/tools/build-manifest/src/main.rs
@@ -205,15 +205,20 @@ fn main() {
//
// Once the old release process is fully decommissioned, the environment variable, all the
// related code in this tool and ./x.py dist hash-and-sign can be removed.
- let legacy = env::var("BUILD_MANIFEST_LEGACY").is_ok();
-
- // Avoid overloading the old server in legacy mode.
- if legacy {
- rayon::ThreadPoolBuilder::new()
- .num_threads(1)
- .build_global()
- .expect("failed to initialize Rayon");
- }
+ let legacy = env::var_os("BUILD_MANIFEST_LEGACY").is_some();
+
+ let num_threads = if legacy {
+ // Avoid overloading the old server in legacy mode.
+ 1
+ } else if let Some(num) = env::var_os("BUILD_MANIFEST_NUM_THREADS") {
+ num.to_str().unwrap().parse().expect("invalid number for BUILD_MANIFEST_NUM_THREADS")
+ } else {
+ num_cpus::get()
+ };
+ rayon::ThreadPoolBuilder::new()
+ .num_threads(num_threads)
+ .build_global()
+ .expect("failed to initialize Rayon");
let mut args = env::args().skip(1);
let input = PathBuf::from(args.next().unwrap());
@@ -221,7 +226,6 @@ fn main() {
let date = args.next().unwrap();
let s3_address = args.next().unwrap();
let channel = args.next().unwrap();
- let monorepo_path = args.next().unwrap();
// Do not ask for a passphrase while manually testing
let mut passphrase = String::new();
@@ -231,7 +235,7 @@ fn main() {
}
Builder {
- versions: Versions::new(&channel, &input, Path::new(&monorepo_path)).unwrap(),
+ versions: Versions::new(&channel, &input).unwrap(),
input,
output,
diff --git a/src/tools/build-manifest/src/versions.rs b/src/tools/build-manifest/src/versions.rs
index 75b6979b54a78..79f2ef8dfc450 100644
--- a/src/tools/build-manifest/src/versions.rs
+++ b/src/tools/build-manifest/src/versions.rs
@@ -1,4 +1,4 @@
-use anyhow::{Context, Error};
+use anyhow::Error;
use flate2::read::GzDecoder;
use std::collections::HashMap;
use std::fs::File;
@@ -7,6 +7,7 @@ use std::path::{Path, PathBuf};
use tar::Archive;
const DEFAULT_TARGET: &str = "x86_64-unknown-linux-gnu";
+const RUSTC_VERSION: &str = include_str!("../../../version");
#[derive(Debug, Hash, Eq, PartialEq, Clone)]
pub(crate) enum PkgType {
@@ -87,26 +88,13 @@ pub(crate) struct VersionInfo {
pub(crate) struct Versions {
channel: String,
- rustc_version: String,
dist_path: PathBuf,
versions: HashMap<PkgType, VersionInfo>,
}
impl Versions {
- pub(crate) fn new(
- channel: &str,
- dist_path: &Path,
- monorepo_root: &Path,
- ) -> Result<Self, Error> {
- Ok(Self {
- channel: channel.into(),
- rustc_version: std::fs::read_to_string(monorepo_root.join("src").join("version"))
- .context("failed to read the rustc version from src/version")?
- .trim()
- .to_string(),
- dist_path: dist_path.into(),
- versions: HashMap::new(),
- })
+ pub(crate) fn new(channel: &str, dist_path: &Path) -> Result<Self, Error> {
+ Ok(Self { channel: channel.into(), dist_path: dist_path.into(), versions: HashMap::new() })
}
pub(crate) fn channel(&self) -> &str {
@@ -184,10 +172,10 @@ impl Versions {
) -> Result<String, Error> {
let component_name = package.tarball_component_name();
let version = match self.channel.as_str() {
- "stable" => self.rustc_version.clone(),
+ "stable" => RUSTC_VERSION.into(),
"beta" => "beta".into(),
"nightly" => "nightly".into(),
- _ => format!("{}-dev", self.rustc_version),
+ _ => format!("{}-dev", RUSTC_VERSION),
};
if package.target_independent() {
@@ -198,6 +186,6 @@ impl Versions {
}
pub(crate) fn rustc_version(&self) -> &str {
- &self.rustc_version
+ RUSTC_VERSION
}
}