diff --git a/Cargo.lock b/Cargo.lock
index 149f02b4b8c94..f44644cffa62a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3559,6 +3559,7 @@ dependencies = [
"flate2",
"libc",
"log",
+ "measureme",
"rustc",
"rustc-demangle",
"rustc_attr",
diff --git a/src/librustc_codegen_llvm/Cargo.toml b/src/librustc_codegen_llvm/Cargo.toml
index e7c0ee5ea763e..c5e862ffc179f 100644
--- a/src/librustc_codegen_llvm/Cargo.toml
+++ b/src/librustc_codegen_llvm/Cargo.toml
@@ -14,6 +14,7 @@ doctest = false
bitflags = "1.0"
flate2 = "1.0"
libc = "0.2"
+measureme = "0.7.1"
log = "0.4"
rustc = { path = "../librustc" }
rustc-demangle = "0.1"
diff --git a/src/librustc_codegen_llvm/back/lto.rs b/src/librustc_codegen_llvm/back/lto.rs
index e3d69fc5c76df..d7297ed41769c 100644
--- a/src/librustc_codegen_llvm/back/lto.rs
+++ b/src/librustc_codegen_llvm/back/lto.rs
@@ -593,7 +593,7 @@ pub(crate) fn run_pass_manager(
} else {
opt_level
};
- write::optimize_with_new_llvm_pass_manager(module, config, opt_level, opt_stage);
+ write::optimize_with_new_llvm_pass_manager(cgcx, module, config, opt_level, opt_stage);
debug!("lto done");
return;
}
diff --git a/src/librustc_codegen_llvm/back/profiling.rs b/src/librustc_codegen_llvm/back/profiling.rs
new file mode 100644
index 0000000000000..d56ddac699b09
--- /dev/null
+++ b/src/librustc_codegen_llvm/back/profiling.rs
@@ -0,0 +1,58 @@
+use measureme::{event_id::SEPARATOR_BYTE, EventId, StringComponent, StringId};
+use rustc_data_structures::profiling::{SelfProfiler, TimingGuard};
+use std::ffi::{c_void, CStr};
+use std::os::raw::c_char;
+use std::sync::Arc;
+
+fn llvm_args_to_string_id(profiler: &SelfProfiler, pass_name: &str, ir_name: &str) -> EventId {
+ let pass_name = profiler.get_or_alloc_cached_string(pass_name);
+ let mut components = vec![StringComponent::Ref(pass_name)];
+ // handle that LazyCallGraph::SCC is a comma separated list within parentheses
+ let parentheses: &[_] = &['(', ')'];
+ let trimed = ir_name.trim_matches(parentheses);
+ for part in trimed.split(", ") {
+ let demangled_ir_name = rustc_demangle::demangle(part).to_string();
+ let ir_name = profiler.get_or_alloc_cached_string(demangled_ir_name);
+ components.push(StringComponent::Value(SEPARATOR_BYTE));
+ components.push(StringComponent::Ref(ir_name));
+ }
+ EventId::from_label(profiler.alloc_string(components.as_slice()))
+}
+
+pub struct LlvmSelfProfiler<'a> {
+ profiler: Arc<SelfProfiler>,
+ stack: Vec<TimingGuard<'a>>,
+ llvm_pass_event_kind: StringId,
+}
+
+impl<'a> LlvmSelfProfiler<'a> {
+ pub fn new(profiler: Arc<SelfProfiler>) -> Self {
+ let llvm_pass_event_kind = profiler.alloc_string("LLVM Pass");
+ Self { profiler, stack: Vec::default(), llvm_pass_event_kind }
+ }
+
+ fn before_pass_callback(&'a mut self, pass_name: &str, ir_name: &str) {
+ let event_id = llvm_args_to_string_id(&self.profiler, pass_name, ir_name);
+
+ self.stack.push(TimingGuard::start(&self.profiler, self.llvm_pass_event_kind, event_id));
+ }
+ fn after_pass_callback(&mut self) {
+ self.stack.pop();
+ }
+}
+
+pub unsafe extern "C" fn selfprofile_before_pass_callback(
+ llvm_self_profiler: *mut c_void,
+ pass_name: *const c_char,
+ ir_name: *const c_char,
+) {
+ let llvm_self_profiler = &mut *(llvm_self_profiler as *mut LlvmSelfProfiler<'_>);
+ let pass_name = CStr::from_ptr(pass_name).to_str().expect("valid UTF-8");
+ let ir_name = CStr::from_ptr(ir_name).to_str().expect("valid UTF-8");
+ llvm_self_profiler.before_pass_callback(pass_name, ir_name);
+}
+
+pub unsafe extern "C" fn selfprofile_after_pass_callback(llvm_self_profiler: *mut c_void) {
+ let llvm_self_profiler = &mut *(llvm_self_profiler as *mut LlvmSelfProfiler<'_>);
+ llvm_self_profiler.after_pass_callback();
+}
diff --git a/src/librustc_codegen_llvm/back/write.rs b/src/librustc_codegen_llvm/back/write.rs
index 9008970847a59..a215ef81bc9eb 100644
--- a/src/librustc_codegen_llvm/back/write.rs
+++ b/src/librustc_codegen_llvm/back/write.rs
@@ -1,6 +1,9 @@
use crate::attributes;
use crate::back::bytecode;
use crate::back::lto::ThinBuffer;
+use crate::back::profiling::{
+ selfprofile_after_pass_callback, selfprofile_before_pass_callback, LlvmSelfProfiler,
+};
use crate::base;
use crate::common;
use crate::consts;
@@ -348,6 +351,7 @@ pub(crate) fn should_use_new_llvm_pass_manager(config: &ModuleConfig) -> bool {
}
pub(crate) unsafe fn optimize_with_new_llvm_pass_manager(
+ cgcx: &CodegenContext<LlvmCodegenBackend>,
module: &ModuleCodegen<ModuleLlvm>,
config: &ModuleConfig,
opt_level: config::OptLevel,
@@ -372,6 +376,13 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager(
None
};
+ let llvm_selfprofiler = if cgcx.prof.llvm_recording_enabled() {
+ let mut llvm_profiler = LlvmSelfProfiler::new(cgcx.prof.get_self_profiler().unwrap());
+ &mut llvm_profiler as *mut _ as *mut c_void
+ } else {
+ std::ptr::null_mut()
+ };
+
// FIXME: NewPM doesn't provide a facility to pass custom InlineParams.
// We would have to add upstream support for this first, before we can support
// config.inline_threshold and our more aggressive default thresholds.
@@ -394,6 +405,9 @@ pub(crate) unsafe fn optimize_with_new_llvm_pass_manager(
sanitizer_options.as_ref(),
pgo_gen_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
pgo_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
+ llvm_selfprofiler,
+ selfprofile_before_pass_callback,
+ selfprofile_after_pass_callback,
);
}
@@ -428,10 +442,15 @@ pub(crate) unsafe fn optimize(
_ if cgcx.opts.cg.linker_plugin_lto.enabled() => llvm::OptStage::PreLinkThinLTO,
_ => llvm::OptStage::PreLinkNoLTO,
};
- optimize_with_new_llvm_pass_manager(module, config, opt_level, opt_stage);
+ optimize_with_new_llvm_pass_manager(cgcx, module, config, opt_level, opt_stage);
return Ok(());
}
+ if cgcx.prof.llvm_recording_enabled() {
+ diag_handler
+ .warn("`-Z self-profile-events = llvm` requires `-Z new-llvm-pass-manager`");
+ }
+
// Create the two optimizing pass managers. These mirror what clang
// does, and are by populated by LLVM's default PassManagerBuilder.
// Each manager has a different set of passes, but they also share
diff --git a/src/librustc_codegen_llvm/lib.rs b/src/librustc_codegen_llvm/lib.rs
index 7aaa70d6ec4d3..b1085ba170330 100644
--- a/src/librustc_codegen_llvm/lib.rs
+++ b/src/librustc_codegen_llvm/lib.rs
@@ -44,6 +44,7 @@ mod back {
pub mod archive;
pub mod bytecode;
pub mod lto;
+ mod profiling;
pub mod write;
}
diff --git a/src/librustc_codegen_llvm/llvm/ffi.rs b/src/librustc_codegen_llvm/llvm/ffi.rs
index f12bfe0e80ace..808094eca91d5 100644
--- a/src/librustc_codegen_llvm/llvm/ffi.rs
+++ b/src/librustc_codegen_llvm/llvm/ffi.rs
@@ -709,6 +709,10 @@ extern "C" {
pub type ModuleBuffer;
}
+pub type SelfProfileBeforePassCallback =
+ unsafe extern "C" fn(*mut c_void, *const c_char, *const c_char);
+pub type SelfProfileAfterPassCallback = unsafe extern "C" fn(*mut c_void);
+
extern "C" {
pub fn LLVMRustInstallFatalErrorHandler();
@@ -1945,6 +1949,9 @@ extern "C" {
SanitizerOptions: Option<&SanitizerOptions>,
PGOGenPath: *const c_char,
PGOUsePath: *const c_char,
+ llvm_selfprofiler: *mut c_void,
+ begin_callback: SelfProfileBeforePassCallback,
+ end_callback: SelfProfileAfterPassCallback,
);
pub fn LLVMRustPrintModule(
M: &'a Module,
diff --git a/src/librustc_data_structures/profiling.rs b/src/librustc_data_structures/profiling.rs
index debda9f0a0a24..1d0ac4f4907d1 100644
--- a/src/librustc_data_structures/profiling.rs
+++ b/src/librustc_data_structures/profiling.rs
@@ -127,6 +127,7 @@ bitflags::bitflags! {
const QUERY_KEYS = 1 << 5;
const FUNCTION_ARGS = 1 << 6;
+ const LLVM = 1 << 7;
const DEFAULT = Self::GENERIC_ACTIVITIES.bits |
Self::QUERY_PROVIDERS.bits |
@@ -150,6 +151,7 @@ const EVENT_FILTERS_BY_NAME: &[(&str, EventFilter)] = &[
("query-keys", EventFilter::QUERY_KEYS),
("function-args", EventFilter::FUNCTION_ARGS),
("args", EventFilter::ARGS),
+ ("llvm", EventFilter::LLVM),
];
/// Something that uniquely identifies a query invocation.
@@ -364,6 +366,15 @@ impl SelfProfilerRef {
pub fn enabled(&self) -> bool {
self.profiler.is_some()
}
+
+ #[inline]
+ pub fn llvm_recording_enabled(&self) -> bool {
+ self.event_filter_mask.contains(EventFilter::LLVM)
+ }
+ #[inline]
+ pub fn get_self_profiler(&self) -> Option<Arc<SelfProfiler>> {
+ self.profiler.clone()
+ }
}
pub struct SelfProfiler {
diff --git a/src/librustc_session/options.rs b/src/librustc_session/options.rs
index a794670d7b8fe..d3163fa356436 100644
--- a/src/librustc_session/options.rs
+++ b/src/librustc_session/options.rs
@@ -940,7 +940,7 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options,
"specifies which kinds of events get recorded by the self profiler;
for example: `-Z self-profile-events=default,query-keys`
all options: none, all, default, generic-activity, query-provider, query-cache-hit
- query-blocked, incr-cache-load, query-keys"),
+ query-blocked, incr-cache-load, query-keys, function-args, args, llvm"),
emit_stack_sizes: bool = (false, parse_bool, [UNTRACKED],
"emits a section containing stack size metadata"),
plt: Option<bool> = (None, parse_opt_bool, [TRACKED],
diff --git a/src/rustllvm/PassWrapper.cpp b/src/rustllvm/PassWrapper.cpp
index 15e2251d76321..2c283149be8bc 100644
--- a/src/rustllvm/PassWrapper.cpp
+++ b/src/rustllvm/PassWrapper.cpp
@@ -640,6 +640,62 @@ LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR,
return LLVMRustResult::Success;
}
+extern "C" typedef void (*LLVMRustSelfProfileBeforePassCallback)(void*, // LlvmSelfProfiler
+ const char*, // pass name
+ const char*); // IR name
+extern "C" typedef void (*LLVMRustSelfProfileAfterPassCallback)(void*); // LlvmSelfProfiler
+
+#if LLVM_VERSION_GE(9, 0)
+
+std::string LLVMRustwrappedIrGetName(const llvm::Any &WrappedIr) {
+ if (any_isa<const Module *>(WrappedIr))
+ return any_cast<const Module *>(WrappedIr)->getName().str();
+ if (any_isa<const Function *>(WrappedIr))
+ return any_cast<const Function *>(WrappedIr)->getName().str();
+ if (any_isa<const Loop *>(WrappedIr))
+ return any_cast<const Loop *>(WrappedIr)->getName().str();
+ if (any_isa<const LazyCallGraph::SCC *>(WrappedIr))
+ return any_cast<const LazyCallGraph::SCC *>(WrappedIr)->getName();
+ return "<UNKNOWN>";
+}
+
+
+void LLVMSelfProfileInitializeCallbacks(
+ PassInstrumentationCallbacks& PIC, void* LlvmSelfProfiler,
+ LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
+ LLVMRustSelfProfileAfterPassCallback AfterPassCallback) {
+ PIC.registerBeforePassCallback([LlvmSelfProfiler, BeforePassCallback](
+ StringRef Pass, llvm::Any Ir) {
+ std::string PassName = Pass.str();
+ std::string IrName = LLVMRustwrappedIrGetName(Ir);
+ BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
+ return true;
+ });
+
+ PIC.registerAfterPassCallback(
+ [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) {
+ AfterPassCallback(LlvmSelfProfiler);
+ });
+
+ PIC.registerAfterPassInvalidatedCallback(
+ [LlvmSelfProfiler, AfterPassCallback](StringRef Pass) {
+ AfterPassCallback(LlvmSelfProfiler);
+ });
+
+ PIC.registerBeforeAnalysisCallback([LlvmSelfProfiler, BeforePassCallback](
+ StringRef Pass, llvm::Any Ir) {
+ std::string PassName = Pass.str();
+ std::string IrName = LLVMRustwrappedIrGetName(Ir);
+ BeforePassCallback(LlvmSelfProfiler, PassName.c_str(), IrName.c_str());
+ });
+
+ PIC.registerAfterAnalysisCallback(
+ [LlvmSelfProfiler, AfterPassCallback](StringRef Pass, llvm::Any Ir) {
+ AfterPassCallback(LlvmSelfProfiler);
+ });
+}
+#endif
+
enum class LLVMRustOptStage {
PreLinkNoLTO,
PreLinkThinLTO,
@@ -666,7 +722,10 @@ LLVMRustOptimizeWithNewPassManager(
bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize,
bool DisableSimplifyLibCalls,
LLVMRustSanitizerOptions *SanitizerOptions,
- const char *PGOGenPath, const char *PGOUsePath) {
+ const char *PGOGenPath, const char *PGOUsePath,
+ void* LlvmSelfProfiler,
+ LLVMRustSelfProfileBeforePassCallback BeforePassCallback,
+ LLVMRustSelfProfileAfterPassCallback AfterPassCallback) {
#if LLVM_VERSION_GE(9, 0)
Module *TheModule = unwrap(ModuleRef);
TargetMachine *TM = unwrap(TMRef);
@@ -685,6 +744,10 @@ LLVMRustOptimizeWithNewPassManager(
StandardInstrumentations SI;
SI.registerCallbacks(PIC);
+ if (LlvmSelfProfiler){
+ LLVMSelfProfileInitializeCallbacks(PIC,LlvmSelfProfiler,BeforePassCallback,AfterPassCallback);
+ }
+
Optional<PGOOptions> PGOOpt;
if (PGOGenPath) {
assert(!PGOUsePath);