Bug 1465709 - Hook rust OOM handler on rustc 1.28. r=froydnj

Bug 1458161 added a rust OOM handler based on an unstable API that was
removed in 1.27, replaced with something that didn't allow to get the
failed allocation size.

Latest 1.28 nightly (2018-06-13) has
rust-lang/rust#50880,
rust-lang/rust#51264 and
rust-lang/rust#51241 merged, which allow to
hook the OOM handler and get the failed allocation size again.

Because this is still an unstable API, we explicitly depend on strict
versions of rustc. We also explicitly error out if automation builds
end up using a rustc version that doesn't allow us to get the allocation
size for rust OOM, because we don't want that to happen without knowing.

UltraBlame original commit: 5182bca90d0609f182d5a7b6b48ed2ffbbce32c2

Author: marco-c

toolkit/crashreporter/nsExceptionHandler.cpp

@@ -117,6 +117,7 @@ using mozilla::ipc::CrashReporterClient;
 
 extern "C" {
   void install_rust_panic_hook();
+  void install_rust_oom_hook();
   bool get_rust_panic_reason(char** reason, size_t* length);
 }
 
@@ -1687,6 +1688,8 @@ nsresult SetExceptionHandler(nsIFile* aXREDirectory,
 
   install_rust_panic_hook();
 
+  install_rust_oom_hook();
+
   InitThreadAnnotation();
 
   return NS_OK;

toolkit/library/gtest/rust/Cargo.toml

@@ -14,6 +14,7 @@ cubeb_pulse_rust = ["gkrust-shared/cubeb_pulse_rust"]
 gecko_debug = ["gkrust-shared/gecko_debug"]
 simd-accel = ["gkrust-shared/simd-accel"]
 oom_with_global_alloc = ["gkrust-shared/oom_with_global_alloc"]
+oom_with_hook = ["gkrust-shared/oom_with_hook"]
 moz_memory = ["gkrust-shared/moz_memory"]
 
 [dependencies]

toolkit/library/rust/Cargo.toml

@@ -14,6 +14,7 @@ cubeb_pulse_rust = ["gkrust-shared/cubeb_pulse_rust"]
 gecko_debug = ["gkrust-shared/gecko_debug"]
 simd-accel = ["gkrust-shared/simd-accel"]
 oom_with_global_alloc = ["gkrust-shared/oom_with_global_alloc"]
+oom_with_hook = ["gkrust-shared/oom_with_hook"]
 moz_memory = ["gkrust-shared/moz_memory"]
 
 [dependencies]

toolkit/library/rust/gkrust-features.mozbuild

@@ -30,3 +30,10 @@ if CONFIG['MOZ_MEMORY']:
 # A string test is not the best thing, but it works well enough here.
 if CONFIG['RUSTC_VERSION'] < "1.27":
     gkrust_features += ['oom_with_global_alloc']
+elif CONFIG['RUSTC_VERSION'] >= "1.28" and CONFIG['RUSTC_VERSION'] < "1.29":
+    gkrust_features += ['oom_with_hook']
+elif not CONFIG['MOZ_AUTOMATION']:
+    # We don't want builds on automation to unwillingly stop annotating OOM
+    # crash reports from rust.
+    error('Builds on automation must use a version of rust that supports OOM '
+          'hooking')

toolkit/library/rust/shared/Cargo.toml

@@ -38,6 +38,7 @@ cubeb_pulse_rust = ["cubeb-sys", "cubeb-pulse"]
 gecko_debug = ["geckoservo/gecko_debug", "nsstring/gecko_debug"]
 simd-accel = ["encoding_c/simd-accel", "encoding_glue/simd-accel"]
 oom_with_global_alloc = []
+oom_with_hook = []
 moz_memory = ["mp4parse_capi/mp4parse_fallible"]
 
 [lib]

toolkit/library/rust/shared/build.rs

@@ -3,6 +3,6 @@ fn main() {
 
 
 
-    #[cfg(feature = "oom_with_global_alloc")]
+    #[cfg(any(feature = "oom_with_global_alloc", feature = "oom_with_hook"))]
     println!("cargo:rustc-env=RUSTC_BOOTSTRAP=1");
 }

toolkit/library/rust/shared/lib.rs

@@ -4,6 +4,7 @@
 
 #![cfg_attr(feature = "oom_with_global_alloc",
             feature(global_allocator, alloc, alloc_system, allocator_api))]
+#![cfg_attr(feature = "oom_with_hook", feature(oom_hook))]
 
 #[cfg(feature="servo")]
 extern crate geckoservo;
@@ -218,3 +219,28 @@ mod global_alloc {
 #[cfg(feature = "oom_with_global_alloc")]
 #[global_allocator]
 static HEAP: global_alloc::GeckoHeap = global_alloc::GeckoHeap;
+
+#[cfg(feature = "oom_with_hook")]
+mod oom_hook {
+    use std::alloc::{Layout, set_oom_hook};
+
+    extern "C" {
+        fn GeckoHandleOOM(size: usize) -> !;
+    }
+
+    pub fn hook(layout: Layout) {
+        unsafe {
+            GeckoHandleOOM(layout.size());
+        }
+    }
+
+    pub fn install() {
+        set_oom_hook(hook);
+    }
+}
+
+#[no_mangle]
+pub extern "C" fn install_rust_oom_hook() {
+    #[cfg(feature = "oom_with_hook")]
+    oom_hook::install();
+}