Do not try to duplicate too big mappings, used by address sanitizer.

Visible with an ASan enabled rr build in tests
nested_detach, nested_detach_kill and nested_detach_wait.

Just nested_detach_kill is fixed with this,
nested_detach and nested_detach_wait then
show "Assertion `preload_pos == string::npos' failed."

1202: =================================================================
1202: ==901358==ERROR: AddressSanitizer: requested allocation size 0x20000000000 (0x20000001000 after adjustments for alignment, red zones etc.) exceeds maximum supported size of 0x10000000000 (thread T0)
1202:      0 0x7faf22267647 in operator new(unsigned long) ../../../../src/libsanitizer/asan/asan_new_delete.cpp:99
1202:      1 0x562cdcc49466 in __gnu_cxx::new_allocator<char>::allocate(unsigned long, void const*) /usr/include/c++/10/ext/new_allocator.h:115
1202:      2 0x562cdcc48ad5 in std::allocator_traits<std::allocator<char> >::allocate(std::allocator<char>&, unsigned long) /usr/include/c++/10/bits/alloc_traits.h:460
1202:      3 0x562cdcc497fd in std::_Vector_base<char, std::allocator<char> >::_M_allocate(unsigned long) /usr/include/c++/10/bits/stl_vector.h:346
1202:      4 0x562cdcc48f34 in std::vector<char, std::allocator<char> >::_M_default_append(unsigned long) /usr/include/c++/10/bits/vector.tcc:635
1202:      5 0x562cdcc488ae in std::vector<char, std::allocator<char> >::resize(unsigned long) /usr/include/c++/10/bits/stl_vector.h:940
1202:      6 0x562cdd087cb8 in copy_mem_mapping /home/bernhard/data/entwicklung/2021/rr/2021-04-25/rr/src/Task.cc:3575
1202:      7 0x562cdd089c23 in rr::Task::dup_from(rr::Task*) /home/bernhard/data/entwicklung/2021/rr/2021-04-25/rr/src/Task.cc:3680
1202:      8 0x562cdce1597b in do_detach_teleport /home/bernhard/data/entwicklung/2021/rr/2021-04-25/rr/src/record_syscall.cc:3355
1202:      9 0x562cdce3c38c in rec_prepare_syscall_arch<rr::X64Arch> /home/bernhard/data/entwicklung/2021/rr/2021-04-25/rr/src/record_syscall.cc:4796
1202:     10 0x562cdce1620e in operator() /home/bernhard/data/entwicklung/2021/rr/2021-04-25/rr/src/record_syscall.cc:4935
1202:     11 0x562cdce49064 in with_converted_registers<rr::Switchable, rr::rec_prepare_syscall_internal(rr::RecordTask*, rr::TaskSyscallState&)::<lambda(const rr::Registers&)> > /home/bernhard/data/entwicklung/2021/rr/2021-04-25/rr/src/Registers.h:604
1202:     12 0x562cdce163eb in rec_prepare_syscall_internal /home/bernhard/data/entwicklung/2021/rr/2021-04-25/rr/src/record_syscall.cc:4933
1202:     13 0x562cdce16496 in rr::rec_prepare_syscall(rr::RecordTask*) /home/bernhard/data/entwicklung/2021/rr/2021-04-25/rr/src/record_syscall.cc:4944
1202:     14 0x562cdcdd7ea1 in rr::RecordSession::syscall_state_changed(rr::RecordTask*, rr::RecordSession::StepState*) /home/bernhard/data/entwicklung/2021/rr/2021-04-25/rr/src/RecordSession.cc:1076
1202:     15 0x562cdcde5f8d in rr::RecordSession::record_step() /home/bernhard/data/entwicklung/2021/rr/2021-04-25/rr/src/RecordSession.cc:2401
1202:     16 0x562cdcdc7a48 in record /home/bernhard/data/entwicklung/2021/rr/2021-04-25/rr/src/RecordCommand.cc:656
1202:     17 0x562cdcdc935a in rr::RecordCommand::run(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >&) /home/bernhard/data/entwicklung/2021/rr/2021-04-25/rr/src/RecordCommand.cc:791
1202:     18 0x562cdd13aa60 in main /home/bernhard/data/entwicklung/2021/rr/2021-04-25/rr/src/main.cc:249
1202:     19 0x7faf21b97d09 in __libc_start_main ../csu/libc-start.c:308
1202:
1202: ==901358==HINT: if you don't care about these errors you may set allocator_may_return_null=1
1202: SUMMARY: AddressSanitizer: allocation-size-too-big ../../../../src/libsanitizer/asan/asan_new_delete.cpp:99 in operator new(unsigned long)
1202: ==901358==ABORTING

Included is also a test to just copy pages that are really in use.
Unfortuantely using "copy_mem_mapping_just_used" with e.g.
`time bin/rr record bin/rr record --nested=detach bin/simple`
increases the running time from just 3 to around 400 seconds.

From the 20TB it looks like there are around 126 pages in use.

Author: bernhardu

src/Task.cc

@@ -3583,6 +3583,60 @@ static void copy_mem_mapping(Task* from, Task* to, const KernelMapping& km) {
   }
 }
 
+#if defined(__x86_64__) && defined(__SANITIZE_ADDRESS__)
+/* https://www.kernel.org/doc/Documentation/vm/pagemap.txt */
+static void copy_mem_mapping_just_used(Task* from, Task* to, const KernelMapping& km)
+{
+  //LOG(error) << "Copying " << km << " size=0x" << std::hex << km.size() << " pagesize=0x" << std::hex << page_size() << std::dec;
+
+  char pagemap_path[PATH_MAX];
+  sprintf(pagemap_path, "/proc/%d/pagemap", from->tid);
+
+  ScopedFd pagemap_fd(pagemap_path, O_RDONLY);
+  if (!pagemap_fd.is_open()) {
+    FATAL() << "Failed to open " << pagemap_path;
+  }
+
+  uint64_t chunk_start = km.start().as_int();
+  uint64_t chunk_size = 0x40000000/*1GB*/;
+
+  off_t read_offset;
+  ssize_t read_size;
+
+  vector<uint64_t> pagemap;
+  pagemap.resize(chunk_size / page_size() * sizeof(uint64_t));
+
+  do {
+      chunk_size = std::min(chunk_size, km.end().as_int() - chunk_start);
+
+      read_offset = chunk_start / page_size() * sizeof(uint64_t);
+      read_size = chunk_size / page_size() * sizeof(uint64_t);
+      //LOG(error) << "read_offset=0x" << std::hex << read_offset << " read_size=0x" << std::hex << read_size << std::dec;
+      if (read_offset != lseek(pagemap_fd, read_offset, SEEK_SET)) {
+        FATAL() << "Could not seek to expected position in " << pagemap_path;
+      }
+
+      if (read_size != read(pagemap_fd, pagemap.data(), read_size)) {
+        FATAL() << "Reading pagemap failed " << pagemap_path;
+      }
+
+      for (size_t i = 0; i < pagemap.size(); i++) {
+        if (pagemap[i] & 0x8000000000000000) { /* Bit  63    page present */
+          //LOG(error) << "addr=0x" << std::hex << chunk_start + i * page_size() << " PFN=0x" << std::hex << pagemap[i] << std::dec;
+          auto page = km.subrange(chunk_start + i * page_size(),
+                                  chunk_start + (i+1) * page_size());
+          //LOG(error) << "Copying page " << page;
+          copy_mem_mapping(from, to, page);
+        }
+      }
+
+      chunk_start += chunk_size;
+  } while (chunk_start < km.end().as_int());
+
+  //LOG(error) << "Finished " << km;
+}
+#endif
+
 static void move_vdso_mapping(AutoRemoteSyscalls &remote, const KernelMapping &km) {
   for (const auto& m : remote.task()->vm()->maps()) {
     if  (m.map.is_vdso() && m.map.start() != km.start()) {
@@ -3669,6 +3723,16 @@ void Task::dup_from(Task *other) {
       create_mapping(this, remote_this, km);
       LOG(debug) << "Copying mapping into " << tid;
       if (!(km.flags() & MAP_SHARED)) {
+#if defined(__x86_64__) && defined(__SANITIZE_ADDRESS__)
+        if (km.flags() & MAP_NORESERVE /*&& km.size() >= 0x100000000*/) {
+          if (1) {
+            LOG(warn) << "Not duplicating too big mapping of ASAN shadow or allocator reserve:" << km;
+          } else {
+            copy_mem_mapping_just_used(other, this, km);
+          }
+          continue;
+        }
+#endif
         copy_mem_mapping(other, this, km);
       }
     }

src/main.cc

@@ -7,6 +7,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <sys/mman.h>
 #include <sys/utsname.h>
 
 #include <sstream>
@@ -205,10 +206,26 @@ bool parse_global_option(std::vector<std::string>& args) {
 
 using namespace rr;
 
+static void prepare_test_copy_mem_mapping_just_used() {
+#if 0
+  char* addr = (char*)0x5a0000000000;
+  void* ret = mmap(addr,
+                   page_size() * 0x40,
+                   PROT_READ | PROT_WRITE,
+                   MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0);
+  if (ret == MAP_FAILED) {
+    DEBUG_ASSERT(0 && "mmap failed.");
+  }
+  memset(addr + page_size() * 0x20, 0xaa, page_size());
+#endif
+}
+
 int main(int argc, char* argv[]) {
   init_random();
   raise_resource_limits();
 
+  prepare_test_copy_mem_mapping_just_used();
+
   vector<string> args;
   for (int i = 1; i < argc; ++i) {
     args.push_back(argv[i]);