From c55b5b2c9f49d23a6063cc6e7a756e22c9cede43 Mon Sep 17 00:00:00 2001
From: John Holdsworth <github@johnholdsworth.com>
Date: Sat, 5 Jul 2025 10:24:51 +0200
Subject: [PATCH 01/21] Multi-threaded disk i/o.

---
 lld/MachO/Config.h   |   1 +
 lld/MachO/Driver.cpp | 111 +++++++++++++++++++++++++++++++++++++++----
 lld/MachO/Options.td |   3 ++
 3 files changed, 106 insertions(+), 9 deletions(-)

diff --git a/lld/MachO/Config.h b/lld/MachO/Config.h
index a01e60efbe761..92c6eb85f4123 100644
--- a/lld/MachO/Config.h
+++ b/lld/MachO/Config.h
@@ -186,6 +186,7 @@ struct Configuration {
   bool interposable = false;
   bool errorForArchMismatch = false;
   bool ignoreAutoLink = false;
+  int readThreads = 0;
   // ld64 allows invalid auto link options as long as the link succeeds. LLD
   // does not, but there are cases in the wild where the invalid linker options
   // exist. This allows users to ignore the specific invalid options in the case
diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index 9eb391c4ee1b9..36626720aa252 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -44,6 +44,7 @@
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Parallel.h"
 #include "llvm/Support/Path.h"
+#include "llvm/Support/Process.h"
 #include "llvm/Support/TarWriter.h"
 #include "llvm/Support/TargetSelect.h"
 #include "llvm/Support/TimeProfiler.h"
@@ -282,11 +283,11 @@ static void saveThinArchiveToRepro(ArchiveFile const *file) {
           ": Archive::children failed: " + toString(std::move(e)));
 }
 
-static InputFile *addFile(StringRef path, LoadType loadType,
-                          bool isLazy = false, bool isExplicit = true,
-                          bool isBundleLoader = false,
-                          bool isForceHidden = false) {
-  std::optional<MemoryBufferRef> buffer = readFile(path);
+static InputFile *deferredAddFile(std::optional<MemoryBufferRef> buffer,
+                                  StringRef path, LoadType loadType,
+                                  bool isLazy = false, bool isExplicit = true,
+                                  bool isBundleLoader = false,
+                                  bool isForceHidden = false) {
   if (!buffer)
     return nullptr;
   MemoryBufferRef mbref = *buffer;
@@ -441,6 +442,14 @@ static InputFile *addFile(StringRef path, LoadType loadType,
   return newFile;
 }
 
+static InputFile *addFile(StringRef path, LoadType loadType,
+                          bool isLazy = false, bool isExplicit = true,
+                          bool isBundleLoader = false,
+                          bool isForceHidden = false) {
+  return deferredAddFile(readFile(path), path, loadType, isLazy, isExplicit,
+                         isBundleLoader, isForceHidden);
+}
+
 static std::vector<StringRef> missingAutolinkWarnings;
 static void addLibrary(StringRef name, bool isNeeded, bool isWeak,
                        bool isReexport, bool isHidden, bool isExplicit,
@@ -564,13 +573,23 @@ void macho::resolveLCLinkerOptions() {
   }
 }
 
-static void addFileList(StringRef path, bool isLazy) {
+typedef struct {
+  StringRef path;
+  std::optional<MemoryBufferRef> buffer;
+} DeferredFile;
+
+static void addFileList(StringRef path, bool isLazy,
+                        std::vector<DeferredFile> &deferredFiles) {
   std::optional<MemoryBufferRef> buffer = readFile(path);
   if (!buffer)
     return;
   MemoryBufferRef mbref = *buffer;
   for (StringRef path : args::getLines(mbref))
-    addFile(rerootPath(path), LoadType::CommandLine, isLazy);
+    if (config->readThreads) {
+      StringRef rrpath = rerootPath(path);
+      deferredFiles.push_back({rrpath, readFile(rrpath)});
+    } else
+      addFile(rerootPath(path), LoadType::CommandLine, isLazy);
 }
 
 // We expect sub-library names of the form "libfoo", which will match a dylib
@@ -1215,13 +1234,68 @@ static void handleSymbolPatterns(InputArgList &args,
     parseSymbolPatternsFile(arg, symbolPatterns);
 }
 
-static void createFiles(const InputArgList &args) {
+// Most input files have been mapped but not yet paged in.
+// This code forces the page-ins on multiple threads so
+// the process is not stalled waiting on disk buffer i/o.
+void multiThreadedPageIn(std::vector<DeferredFile> &deferred, int nthreads) {
+#ifndef _WIN32
+  typedef struct {
+    std::vector<DeferredFile> &deferred;
+    size_t counter, total, pageSize;
+    pthread_mutex_t mutex;
+  } PageInState;
+  PageInState state = {deferred, 0, 0,
+                       llvm::sys::Process::getPageSizeEstimate(),
+                       pthread_mutex_t()};
+  pthread_mutex_init(&state.mutex, NULL);
+
+  pthread_t running[200];
+  int maxthreads = sizeof running / sizeof running[0];
+  if (nthreads > maxthreads)
+    nthreads = maxthreads;
+
+  for (int t = 0; t < nthreads; t++)
+    pthread_create(
+        &running[t], nullptr,
+        [](void *ptr) -> void * {
+          PageInState &state = *(PageInState *)ptr;
+          static int total = 0;
+          while (true) {
+            pthread_mutex_lock(&state.mutex);
+            if (state.counter >= state.deferred.size()) {
+              pthread_mutex_unlock(&state.mutex);
+              return nullptr;
+            }
+            DeferredFile &add = state.deferred[state.counter];
+            state.counter += 1;
+            pthread_mutex_unlock(&state.mutex);
+
+            int t = 0; // Reference each page to load it into memory.
+            for (const char *page = add.buffer->getBuffer().data(),
+                            *end = page + add.buffer->getBuffer().size();
+                 page < end; page += state.pageSize)
+              t += *page;
+            state.total += t; // Avoids whole section being optimised out.
+          }
+        },
+        &state);
+
+  for (int t = 0; t < nthreads; t++)
+    pthread_join(running[t], nullptr);
+
+  pthread_mutex_destroy(&state.mutex);
+#endif
+}
+
+void createFiles(const InputArgList &args) {
   TimeTraceScope timeScope("Load input files");
   // This loop should be reserved for options whose exact ordering matters.
   // Other options should be handled via filtered() and/or getLastArg().
   bool isLazy = false;
   // If we've processed an opening --start-lib, without a matching --end-lib
   bool inLib = false;
+  std::vector<DeferredFile> deferredFiles;
+
   for (const Arg *arg : args) {
     const Option &opt = arg->getOption();
     warnIfDeprecatedOption(opt);
@@ -1229,6 +1303,11 @@ static void createFiles(const InputArgList &args) {
 
     switch (opt.getID()) {
     case OPT_INPUT:
+      if (config->readThreads) {
+        StringRef rrpath = rerootPath(arg->getValue());
+        deferredFiles.push_back({rrpath, readFile(rrpath)});
+        break;
+      }
       addFile(rerootPath(arg->getValue()), LoadType::CommandLine, isLazy);
       break;
     case OPT_needed_library:
@@ -1249,7 +1328,7 @@ static void createFiles(const InputArgList &args) {
         dylibFile->forceWeakImport = true;
       break;
     case OPT_filelist:
-      addFileList(arg->getValue(), isLazy);
+      addFileList(arg->getValue(), isLazy, deferredFiles);
       break;
     case OPT_force_load:
       addFile(rerootPath(arg->getValue()), LoadType::CommandLineForce);
@@ -1295,6 +1374,12 @@ static void createFiles(const InputArgList &args) {
       break;
     }
   }
+
+  if (config->readThreads) {
+    multiThreadedPageIn(deferredFiles, config->readThreads);
+    for (auto &add : deferredFiles)
+      deferredAddFile(add.buffer, add.path, LoadType::CommandLine, isLazy);
+  }
 }
 
 static void gatherInputSections() {
@@ -1687,6 +1772,14 @@ bool link(ArrayRef<const char *> argsArr, llvm::raw_ostream &stdoutOS,
     }
   }
 
+  if (auto *arg = args.getLastArg(OPT_read_threads)) {
+    StringRef v(arg->getValue());
+    unsigned threads = 0;
+    if (!llvm::to_integer(v, threads, 0) || threads < 0)
+      error(arg->getSpelling() + ": expected a positive integer, but got '" +
+            arg->getValue() + "'");
+    config->readThreads = threads;
+  }
   if (auto *arg = args.getLastArg(OPT_threads_eq)) {
     StringRef v(arg->getValue());
     unsigned threads = 0;
diff --git a/lld/MachO/Options.td b/lld/MachO/Options.td
index 4f0602f59812b..3dc98fccc1b7b 100644
--- a/lld/MachO/Options.td
+++ b/lld/MachO/Options.td
@@ -396,6 +396,9 @@ def dead_strip : Flag<["-"], "dead_strip">,
 def interposable : Flag<["-"], "interposable">,
     HelpText<"Indirects access to all exported symbols in an image">,
     Group<grp_opts>;
+def read_threads : Joined<["--"], "read-threads=">,
+    HelpText<"Number of threads to use paging in files.">,
+    Group<grp_lld>;
 def order_file : Separate<["-"], "order_file">,
     MetaVarName<"<file>">,
     HelpText<"Layout functions and data according to specification in <file>">,

From 3d11a33599246bbf5e358b554489aeae854ed7be Mon Sep 17 00:00:00 2001
From: John Holdsworth <github@johnholdsworth.com>
Date: Sun, 6 Jul 2025 10:05:38 +0200
Subject: [PATCH 02/21] Afterthoughts.

---
 lld/MachO/Driver.cpp | 82 +++++++++++++++++++++++---------------------
 1 file changed, 43 insertions(+), 39 deletions(-)

diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index 36626720aa252..5b9f9cc2939bd 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -283,11 +283,11 @@ static void saveThinArchiveToRepro(ArchiveFile const *file) {
           ": Archive::children failed: " + toString(std::move(e)));
 }
 
-static InputFile *deferredAddFile(std::optional<MemoryBufferRef> buffer,
-                                  StringRef path, LoadType loadType,
-                                  bool isLazy = false, bool isExplicit = true,
-                                  bool isBundleLoader = false,
-                                  bool isForceHidden = false) {
+static InputFile *processFile(std::optional<MemoryBufferRef> buffer,
+                              StringRef path, LoadType loadType,
+                              bool isLazy = false, bool isExplicit = true,
+                              bool isBundleLoader = false,
+                              bool isForceHidden = false) {
   if (!buffer)
     return nullptr;
   MemoryBufferRef mbref = *buffer;
@@ -446,8 +446,24 @@ static InputFile *addFile(StringRef path, LoadType loadType,
                           bool isLazy = false, bool isExplicit = true,
                           bool isBundleLoader = false,
                           bool isForceHidden = false) {
-  return deferredAddFile(readFile(path), path, loadType, isLazy, isExplicit,
-                         isBundleLoader, isForceHidden);
+  return processFile(readFile(path), path, loadType, isLazy, isExplicit,
+                     isBundleLoader, isForceHidden);
+}
+
+typedef struct {
+  StringRef path;
+  LoadType loadType;
+  bool isLazy;
+  std::optional<MemoryBufferRef> buffer;
+} DeferredFile;
+
+static void deferFile(StringRef path, LoadType loadType, bool isLazy,
+                      std::vector<DeferredFile> &deferred) {
+  std::optional<MemoryBufferRef> buffer = readFile(path);
+  if (config->readThreads)
+    deferred.push_back({path, loadType, isLazy, buffer});
+  else
+    processFile(buffer, path, loadType, isLazy);
 }
 
 static std::vector<StringRef> missingAutolinkWarnings;
@@ -573,11 +589,6 @@ void macho::resolveLCLinkerOptions() {
   }
 }
 
-typedef struct {
-  StringRef path;
-  std::optional<MemoryBufferRef> buffer;
-} DeferredFile;
-
 static void addFileList(StringRef path, bool isLazy,
                         std::vector<DeferredFile> &deferredFiles) {
   std::optional<MemoryBufferRef> buffer = readFile(path);
@@ -585,11 +596,7 @@ static void addFileList(StringRef path, bool isLazy,
     return;
   MemoryBufferRef mbref = *buffer;
   for (StringRef path : args::getLines(mbref))
-    if (config->readThreads) {
-      StringRef rrpath = rerootPath(path);
-      deferredFiles.push_back({rrpath, readFile(rrpath)});
-    } else
-      addFile(rerootPath(path), LoadType::CommandLine, isLazy);
+    deferFile(rerootPath(path), LoadType::CommandLine, isLazy, deferredFiles);
 }
 
 // We expect sub-library names of the form "libfoo", which will match a dylib
@@ -1239,43 +1246,44 @@ static void handleSymbolPatterns(InputArgList &args,
 // the process is not stalled waiting on disk buffer i/o.
 void multiThreadedPageIn(std::vector<DeferredFile> &deferred, int nthreads) {
 #ifndef _WIN32
+#define MaxReadThreads 200
   typedef struct {
     std::vector<DeferredFile> &deferred;
-    size_t counter, total, pageSize;
+    size_t counter, bytes, total, pageSize;
     pthread_mutex_t mutex;
   } PageInState;
-  PageInState state = {deferred, 0, 0,
-                       llvm::sys::Process::getPageSizeEstimate(),
-                       pthread_mutex_t()};
+  PageInState state = {
+      deferred,         0, 0, 0, llvm::sys::Process::getPageSizeEstimate(),
+      pthread_mutex_t()};
   pthread_mutex_init(&state.mutex, NULL);
 
-  pthread_t running[200];
-  int maxthreads = sizeof running / sizeof running[0];
-  if (nthreads > maxthreads)
-    nthreads = maxthreads;
+  pthread_t running[MaxReadThreads];
+  if (nthreads > MaxReadThreads)
+    nthreads = MaxReadThreads;
 
   for (int t = 0; t < nthreads; t++)
     pthread_create(
         &running[t], nullptr,
         [](void *ptr) -> void * {
           PageInState &state = *(PageInState *)ptr;
-          static int total = 0;
           while (true) {
             pthread_mutex_lock(&state.mutex);
             if (state.counter >= state.deferred.size()) {
               pthread_mutex_unlock(&state.mutex);
               return nullptr;
             }
-            DeferredFile &add = state.deferred[state.counter];
+            DeferredFile &file = state.deferred[state.counter];
             state.counter += 1;
             pthread_mutex_unlock(&state.mutex);
 
+            const char *page = file.buffer->getBuffer().data(),
+                       *end = page + file.buffer->getBuffer().size();
+            state.bytes += end - page;
+
             int t = 0; // Reference each page to load it into memory.
-            for (const char *page = add.buffer->getBuffer().data(),
-                            *end = page + add.buffer->getBuffer().size();
-                 page < end; page += state.pageSize)
+            for (; page < end; page += state.pageSize)
               t += *page;
-            state.total += t; // Avoids whole section being optimised out.
+            state.total += t; // Avoids the loop being optimised out.
           }
         },
         &state);
@@ -1303,12 +1311,8 @@ void createFiles(const InputArgList &args) {
 
     switch (opt.getID()) {
     case OPT_INPUT:
-      if (config->readThreads) {
-        StringRef rrpath = rerootPath(arg->getValue());
-        deferredFiles.push_back({rrpath, readFile(rrpath)});
-        break;
-      }
-      addFile(rerootPath(arg->getValue()), LoadType::CommandLine, isLazy);
+      deferFile(rerootPath(arg->getValue()), LoadType::CommandLine, isLazy,
+                deferredFiles);
       break;
     case OPT_needed_library:
       if (auto *dylibFile = dyn_cast_or_null<DylibFile>(
@@ -1377,8 +1381,8 @@ void createFiles(const InputArgList &args) {
 
   if (config->readThreads) {
     multiThreadedPageIn(deferredFiles, config->readThreads);
-    for (auto &add : deferredFiles)
-      deferredAddFile(add.buffer, add.path, LoadType::CommandLine, isLazy);
+    for (auto &file : deferredFiles)
+      processFile(file.buffer, file.path, file.loadType, file.isLazy);
   }
 }
 

From 02fb145b4b2f2710c59ea750e0b14292abc8c58c Mon Sep 17 00:00:00 2001
From: John Holdsworth <github@johnholdsworth.com>
Date: Sun, 6 Jul 2025 18:09:19 +0200
Subject: [PATCH 03/21] multiThreadedPageIn of library archives.

---
 lld/MachO/Driver.cpp | 149 +++++++++++++++++++++++--------------------
 1 file changed, 79 insertions(+), 70 deletions(-)

diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index 5b9f9cc2939bd..bacbe24fb4434 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -283,6 +283,70 @@ static void saveThinArchiveToRepro(ArchiveFile const *file) {
           ": Archive::children failed: " + toString(std::move(e)));
 }
 
+typedef struct {
+  StringRef path;
+  bool isLazy;
+  std::optional<MemoryBufferRef> buffer;
+  const char *start;
+  size_t size;
+} DeferredFile;
+
+// Most input files have been mapped but not yet paged in.
+// This code forces the page-ins on multiple threads so
+// the process is not stalled waiting on disk buffer i/o.
+static void multiThreadedPageIn(std::vector<DeferredFile> &deferred) {
+#ifndef _WIN32
+#define MaxReadThreads 200
+  typedef struct {
+    std::vector<DeferredFile> &deferred;
+    size_t counter, total, pageSize;
+    pthread_mutex_t mutex;
+  } PageInState;
+  PageInState state = {deferred, 0, 0,
+                       llvm::sys::Process::getPageSizeEstimate(),
+                       pthread_mutex_t()};
+  static size_t totalBytes;
+
+  pthread_t running[MaxReadThreads];
+  if (config->readThreads > MaxReadThreads)
+    config->readThreads = MaxReadThreads;
+  pthread_mutex_init(&state.mutex, NULL);
+
+  for (int t = 0; t < config->readThreads; t++)
+    pthread_create(
+        &running[t], nullptr,
+        [](void *ptr) -> void * {
+          PageInState &state = *(PageInState *)ptr;
+          while (true) {
+            pthread_mutex_lock(&state.mutex);
+            if (state.counter >= state.deferred.size()) {
+              pthread_mutex_unlock(&state.mutex);
+              return nullptr;
+            }
+            DeferredFile &file = state.deferred[state.counter];
+            state.counter += 1;
+            pthread_mutex_unlock(&state.mutex);
+
+            const char *page = file.start, *end = page + file.size;
+            totalBytes += end - page;
+
+            int t = 0; // Reference each page to load it into memory.
+            for (; page < end; page += state.pageSize)
+              t += *page;
+            state.total += t; // Avoids the loop being optimised out.
+          }
+        },
+        &state);
+
+  for (int t = 0; t < config->readThreads; t++)
+    pthread_join(running[t], nullptr);
+
+  pthread_mutex_destroy(&state.mutex);
+  if (getenv("LLD_MULTI_THREAD_PAGE"))
+    printf("multiThreadedPageIn %ld/%ld\n", totalBytes, deferred.size());
+#endif
+}
+
 static InputFile *processFile(std::optional<MemoryBufferRef> buffer,
                               StringRef path, LoadType loadType,
                               bool isLazy = false, bool isExplicit = true,
@@ -367,6 +431,7 @@ static InputFile *processFile(std::optional<MemoryBufferRef> buffer,
       // we already found that it contains an ObjC symbol.
       if (readFile(path)) {
         Error e = Error::success();
+        std::vector<DeferredFile> deferredFiles;
         for (const object::Archive::Child &c : file->getArchive().children(e)) {
           Expected<MemoryBufferRef> mb = c.getMemoryBufferRef();
           if (!mb) {
@@ -380,6 +445,9 @@ static InputFile *processFile(std::optional<MemoryBufferRef> buffer,
             continue;
           }
 
+          deferredFiles.push_back({path, isLazy, std::nullopt,
+                                   mb->getBuffer().data(),
+                                   mb->getBuffer().size()});
           if (!hasObjCSection(*mb))
             continue;
           if (Error e = file->fetch(c, "-ObjC"))
@@ -389,6 +457,8 @@ static InputFile *processFile(std::optional<MemoryBufferRef> buffer,
         if (e)
           error(toString(file) +
                 ": Archive::children failed: " + toString(std::move(e)));
+        if (config->readThreads && deferredFiles.size() > 1)
+          multiThreadedPageIn(deferredFiles);
       }
     }
     file->addLazySymbols();
@@ -450,20 +520,14 @@ static InputFile *addFile(StringRef path, LoadType loadType,
                      isBundleLoader, isForceHidden);
 }
 
-typedef struct {
-  StringRef path;
-  LoadType loadType;
-  bool isLazy;
-  std::optional<MemoryBufferRef> buffer;
-} DeferredFile;
-
-static void deferFile(StringRef path, LoadType loadType, bool isLazy,
+static void deferFile(StringRef path, bool isLazy,
                       std::vector<DeferredFile> &deferred) {
   std::optional<MemoryBufferRef> buffer = readFile(path);
   if (config->readThreads)
-    deferred.push_back({path, loadType, isLazy, buffer});
+    deferred.push_back({path, isLazy, buffer, buffer->getBuffer().data(),
+                        buffer->getBuffer().size()});
   else
-    processFile(buffer, path, loadType, isLazy);
+    processFile(buffer, path, LoadType::CommandLine, isLazy);
 }
 
 static std::vector<StringRef> missingAutolinkWarnings;
@@ -596,7 +660,7 @@ static void addFileList(StringRef path, bool isLazy,
     return;
   MemoryBufferRef mbref = *buffer;
   for (StringRef path : args::getLines(mbref))
-    deferFile(rerootPath(path), LoadType::CommandLine, isLazy, deferredFiles);
+    deferFile(rerootPath(path), isLazy, deferredFiles);
 }
 
 // We expect sub-library names of the form "libfoo", which will match a dylib
@@ -1241,61 +1305,7 @@ static void handleSymbolPatterns(InputArgList &args,
     parseSymbolPatternsFile(arg, symbolPatterns);
 }
 
-// Most input files have been mapped but not yet paged in.
-// This code forces the page-ins on multiple threads so
-// the process is not stalled waiting on disk buffer i/o.
-void multiThreadedPageIn(std::vector<DeferredFile> &deferred, int nthreads) {
-#ifndef _WIN32
-#define MaxReadThreads 200
-  typedef struct {
-    std::vector<DeferredFile> &deferred;
-    size_t counter, bytes, total, pageSize;
-    pthread_mutex_t mutex;
-  } PageInState;
-  PageInState state = {
-      deferred,         0, 0, 0, llvm::sys::Process::getPageSizeEstimate(),
-      pthread_mutex_t()};
-  pthread_mutex_init(&state.mutex, NULL);
-
-  pthread_t running[MaxReadThreads];
-  if (nthreads > MaxReadThreads)
-    nthreads = MaxReadThreads;
-
-  for (int t = 0; t < nthreads; t++)
-    pthread_create(
-        &running[t], nullptr,
-        [](void *ptr) -> void * {
-          PageInState &state = *(PageInState *)ptr;
-          while (true) {
-            pthread_mutex_lock(&state.mutex);
-            if (state.counter >= state.deferred.size()) {
-              pthread_mutex_unlock(&state.mutex);
-              return nullptr;
-            }
-            DeferredFile &file = state.deferred[state.counter];
-            state.counter += 1;
-            pthread_mutex_unlock(&state.mutex);
-
-            const char *page = file.buffer->getBuffer().data(),
-                       *end = page + file.buffer->getBuffer().size();
-            state.bytes += end - page;
-
-            int t = 0; // Reference each page to load it into memory.
-            for (; page < end; page += state.pageSize)
-              t += *page;
-            state.total += t; // Avoids the loop being optimised out.
-          }
-        },
-        &state);
-
-  for (int t = 0; t < nthreads; t++)
-    pthread_join(running[t], nullptr);
-
-  pthread_mutex_destroy(&state.mutex);
-#endif
-}
-
-void createFiles(const InputArgList &args) {
+static void createFiles(const InputArgList &args) {
   TimeTraceScope timeScope("Load input files");
   // This loop should be reserved for options whose exact ordering matters.
   // Other options should be handled via filtered() and/or getLastArg().
@@ -1311,8 +1321,7 @@ void createFiles(const InputArgList &args) {
 
     switch (opt.getID()) {
     case OPT_INPUT:
-      deferFile(rerootPath(arg->getValue()), LoadType::CommandLine, isLazy,
-                deferredFiles);
+      deferFile(rerootPath(arg->getValue()), isLazy, deferredFiles);
       break;
     case OPT_needed_library:
       if (auto *dylibFile = dyn_cast_or_null<DylibFile>(
@@ -1380,9 +1389,9 @@ void createFiles(const InputArgList &args) {
   }
 
   if (config->readThreads) {
-    multiThreadedPageIn(deferredFiles, config->readThreads);
+    multiThreadedPageIn(deferredFiles);
     for (auto &file : deferredFiles)
-      processFile(file.buffer, file.path, file.loadType, file.isLazy);
+      processFile(file.buffer, file.path, LoadType::CommandLine, file.isLazy);
   }
 }
 

From a8eeead77a34e1072d8a1f7eabcc983388e303a4 Mon Sep 17 00:00:00 2001
From: John Holdsworth <github@johnholdsworth.com>
Date: Tue, 8 Jul 2025 14:46:19 +0200
Subject: [PATCH 04/21] Multi-thread i/o in background.

---
 lld/MachO/Driver.cpp | 104 ++++++++++++++++++++++++++++++-------------
 1 file changed, 73 insertions(+), 31 deletions(-)

diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index bacbe24fb4434..ba5ac7eff585a 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -290,27 +290,26 @@ typedef struct {
   const char *start;
   size_t size;
 } DeferredFile;
+typedef std::vector<DeferredFile> DeferredFiles;
+
+#ifndef _WIN32
+typedef struct {
+  DeferredFiles deferred;
+  size_t counter, total, pageSize;
+  pthread_mutex_t mutex;
+} PageInState;
 
 // Most input files have been mapped but not yet paged in.
 // This code forces the page-ins on multiple threads so
 // the process is not stalled waiting on disk buffer i/o.
-static void multiThreadedPageIn(std::vector<DeferredFile> &deferred) {
-#ifndef _WIN32
+static void multiThreadedPageInBackground(PageInState *state) {
 #define MaxReadThreads 200
-  typedef struct {
-    std::vector<DeferredFile> &deferred;
-    size_t counter, total, pageSize;
-    pthread_mutex_t mutex;
-  } PageInState;
-  PageInState state = {deferred, 0, 0,
-                       llvm::sys::Process::getPageSizeEstimate(),
-                       pthread_mutex_t()};
   static size_t totalBytes;
 
   pthread_t running[MaxReadThreads];
   if (config->readThreads > MaxReadThreads)
     config->readThreads = MaxReadThreads;
-  pthread_mutex_init(&state.mutex, NULL);
+  pthread_mutex_init(&state->mutex, nullptr);
 
   for (int t = 0; t < config->readThreads; t++)
     pthread_create(
@@ -336,20 +335,49 @@ static void multiThreadedPageIn(std::vector<DeferredFile> &deferred) {
             state.total += t; // Avoids the loop being optimised out.
           }
         },
-        &state);
+        state);
 
   for (int t = 0; t < config->readThreads; t++)
     pthread_join(running[t], nullptr);
 
-  pthread_mutex_destroy(&state.mutex);
+  pthread_mutex_destroy(&state->mutex);
   if (getenv("LLD_MULTI_THREAD_PAGE"))
-    printf("multiThreadedPageIn %ld/%ld\n", totalBytes, deferred.size());
+    printf("multiThreadedPageIn %ld/%ld\n", totalBytes, state->deferred.size());
+}
+#endif
+
+static void multiThreadedPageIn(DeferredFiles deferred) {
+#ifndef _WIN32
+  static pthread_t running;
+  static pthread_mutex_t busy;
+
+  if (running)
+    pthread_join(running, nullptr);
+  else
+    pthread_mutex_init(&busy, nullptr);
+
+  PageInState *state =
+      new PageInState{deferred, 0, 0, llvm::sys::Process::getPageSizeEstimate(),
+                      pthread_mutex_t()};
+
+  pthread_mutex_lock(&busy);
+  pthread_create(
+      &running, nullptr,
+      [](void *ptr) -> void * {
+        PageInState *state = (PageInState *)ptr;
+        multiThreadedPageInBackground(state);
+        pthread_mutex_unlock(&busy);
+        delete state;
+        return nullptr;
+      },
+      state);
 #endif
 }
 
 static InputFile *processFile(std::optional<MemoryBufferRef> buffer,
-                              StringRef path, LoadType loadType,
-                              bool isLazy = false, bool isExplicit = true,
+                              DeferredFiles *archiveContents, StringRef path,
+                              LoadType loadType, bool isLazy = false,
+                              bool isExplicit = true,
                               bool isBundleLoader = false,
                               bool isForceHidden = false) {
   if (!buffer)
@@ -431,7 +459,6 @@ static InputFile *processFile(std::optional<MemoryBufferRef> buffer,
       // we already found that it contains an ObjC symbol.
       if (readFile(path)) {
         Error e = Error::success();
-        std::vector<DeferredFile> deferredFiles;
         for (const object::Archive::Child &c : file->getArchive().children(e)) {
           Expected<MemoryBufferRef> mb = c.getMemoryBufferRef();
           if (!mb) {
@@ -445,9 +472,10 @@ static InputFile *processFile(std::optional<MemoryBufferRef> buffer,
             continue;
           }
 
-          deferredFiles.push_back({path, isLazy, std::nullopt,
-                                   mb->getBuffer().data(),
-                                   mb->getBuffer().size()});
+          if (archiveContents)
+            archiveContents->push_back({path, isLazy, std::nullopt,
+                                        mb->getBuffer().data(),
+                                        mb->getBuffer().size()});
           if (!hasObjCSection(*mb))
             continue;
           if (Error e = file->fetch(c, "-ObjC"))
@@ -457,11 +485,10 @@ static InputFile *processFile(std::optional<MemoryBufferRef> buffer,
         if (e)
           error(toString(file) +
                 ": Archive::children failed: " + toString(std::move(e)));
-        if (config->readThreads && deferredFiles.size() > 1)
-          multiThreadedPageIn(deferredFiles);
       }
     }
-    file->addLazySymbols();
+    if (!archiveContents || archiveContents->empty())
+      file->addLazySymbols();
     loadedArchives[path] = ArchiveFileInfo{file, isCommandLineLoad};
     newFile = file;
     break;
@@ -516,18 +543,17 @@ static InputFile *addFile(StringRef path, LoadType loadType,
                           bool isLazy = false, bool isExplicit = true,
                           bool isBundleLoader = false,
                           bool isForceHidden = false) {
-  return processFile(readFile(path), path, loadType, isLazy, isExplicit,
-                     isBundleLoader, isForceHidden);
+  return processFile(readFile(path), nullptr, path, loadType, isLazy,
+                     isExplicit, isBundleLoader, isForceHidden);
 }
 
-static void deferFile(StringRef path, bool isLazy,
-                      std::vector<DeferredFile> &deferred) {
+static void deferFile(StringRef path, bool isLazy, DeferredFiles &deferred) {
   std::optional<MemoryBufferRef> buffer = readFile(path);
   if (config->readThreads)
     deferred.push_back({path, isLazy, buffer, buffer->getBuffer().data(),
                         buffer->getBuffer().size()});
   else
-    processFile(buffer, path, LoadType::CommandLine, isLazy);
+    processFile(buffer, nullptr, path, LoadType::CommandLine, isLazy);
 }
 
 static std::vector<StringRef> missingAutolinkWarnings;
@@ -654,7 +680,7 @@ void macho::resolveLCLinkerOptions() {
 }
 
 static void addFileList(StringRef path, bool isLazy,
-                        std::vector<DeferredFile> &deferredFiles) {
+                        DeferredFiles &deferredFiles) {
   std::optional<MemoryBufferRef> buffer = readFile(path);
   if (!buffer)
     return;
@@ -1312,7 +1338,7 @@ static void createFiles(const InputArgList &args) {
   bool isLazy = false;
   // If we've processed an opening --start-lib, without a matching --end-lib
   bool inLib = false;
-  std::vector<DeferredFile> deferredFiles;
+  DeferredFiles deferredFiles;
 
   for (const Arg *arg : args) {
     const Option &opt = arg->getOption();
@@ -1390,8 +1416,24 @@ static void createFiles(const InputArgList &args) {
 
   if (config->readThreads) {
     multiThreadedPageIn(deferredFiles);
+
+    DeferredFiles archiveContents;
+    std::vector<ArchiveFile *> archives;
     for (auto &file : deferredFiles)
-      processFile(file.buffer, file.path, LoadType::CommandLine, file.isLazy);
+      if (ArchiveFile *archive = dyn_cast<ArchiveFile>(
+              processFile(file.buffer, &archiveContents, file.path,
+                          LoadType::CommandLine, file.isLazy)))
+        archives.push_back(archive);
+
+    if (!archiveContents.empty()) {
+      multiThreadedPageIn(archiveContents);
+      for (auto *archive : archives)
+        archive->addLazySymbols();
+    }
+
+    // flush threads
+    deferredFiles.clear();
+    multiThreadedPageIn(deferredFiles);
   }
 }
 

From 55e26a80be5073f2b7761cc9515cebe16e43772c Mon Sep 17 00:00:00 2001
From: John Holdsworth <github@johnholdsworth.com>
Date: Wed, 9 Jul 2025 07:31:49 +0200
Subject: [PATCH 05/21] Response to first review.

---
 lld/MachO/Driver.cpp | 138 ++++++++++++++++++-------------------------
 lld/MachO/Options.td |   2 +-
 2 files changed, 60 insertions(+), 80 deletions(-)

diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index ba5ac7eff585a..4f3b562668ab1 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -47,6 +47,7 @@
 #include "llvm/Support/Process.h"
 #include "llvm/Support/TarWriter.h"
 #include "llvm/Support/TargetSelect.h"
+#include "llvm/Support/Threading.h"
 #include "llvm/Support/TimeProfiler.h"
 #include "llvm/TargetParser/Host.h"
 #include "llvm/TextAPI/Architecture.h"
@@ -283,95 +284,75 @@ static void saveThinArchiveToRepro(ArchiveFile const *file) {
           ": Archive::children failed: " + toString(std::move(e)));
 }
 
-typedef struct {
+class DeferredFile {
+public:
   StringRef path;
   bool isLazy;
   std::optional<MemoryBufferRef> buffer;
   const char *start;
   size_t size;
-} DeferredFile;
-typedef std::vector<DeferredFile> DeferredFiles;
+};
+using DeferredFiles = std::vector<DeferredFile>;
 
-#ifndef _WIN32
-typedef struct {
+class PageInState {
   DeferredFiles deferred;
-  size_t counter, total, pageSize;
-  pthread_mutex_t mutex;
-} PageInState;
-
-// Most input files have been mapped but not yet paged in.
-// This code forces the page-ins on multiple threads so
-// the process is not stalled waiting on disk buffer i/o.
-static void multiThreadedPageInBackground(PageInState *state) {
-#define MaxReadThreads 200
-  static size_t totalBytes;
-
-  pthread_t running[MaxReadThreads];
-  if (config->readThreads > MaxReadThreads)
-    config->readThreads = MaxReadThreads;
-  pthread_mutex_init(&state->mutex, nullptr);
-
-  for (int t = 0; t < config->readThreads; t++)
-    pthread_create(
-        &running[t], nullptr,
-        [](void *ptr) -> void * {
-          PageInState &state = *(PageInState *)ptr;
-          while (true) {
-            pthread_mutex_lock(&state.mutex);
-            if (state.counter >= state.deferred.size()) {
-              pthread_mutex_unlock(&state.mutex);
-              return nullptr;
-            }
-            DeferredFile &file = state.deferred[state.counter];
-            state.counter += 1;
-            pthread_mutex_unlock(&state.mutex);
-
-            const char *page = file.start, *end = page + file.size;
-            totalBytes += end - page;
-
-            int t = 0; // Reference each page to load it into memory.
-            for (; page < end; page += state.pageSize)
-              t += *page;
-            state.total += t; // Avoids the loop being optimised out.
-          }
-        },
-        state);
+  size_t counter = 0, total = 0, pageSize;
+  std::mutex mutex, *busy;
+
+public:
+  PageInState(DeferredFiles &deferred, std::mutex *busy) {
+    this->deferred = deferred;
+    this->busy = busy;
+    pageSize = llvm::sys::Process::getPageSizeEstimate();
+  }
+
+  // Most input files have been mapped but not yet paged in.
+  // This code forces the page-ins on multiple threads so
+  // the process is not stalled waiting on disk buffer i/o.
+  void multiThreadedPageInBackground() {
+    static size_t totalBytes;
+
+    parallelFor(0, config->readThreads, [&](size_t I) {
+      while (true) {
+        mutex.lock();
+        if (counter >= deferred.size()) {
+          mutex.unlock();
+          return;
+        }
+        DeferredFile &file = deferred[counter];
+        totalBytes += file.size;
+        counter += 1;
+        mutex.unlock();
+
+        int t = 0; // Reference each page to load it into memory.
+        for (const char *page = file.start, *end = page + file.size; page < end;
+             page += pageSize)
+          t += *page;
+        total += t; // Avoids the loop being optimised out.
+      }
+    });
 
-  for (int t = 0; t < config->readThreads; t++)
-    pthread_join(running[t], nullptr);
+    if (getenv("LLD_MULTI_THREAD_PAGE"))
+      llvm::dbgs() << "multiThreadedPageIn " << totalBytes << "/"
+                   << deferred.size() << "\n";
 
-  pthread_mutex_destroy(&state->mutex);
-  if (getenv("LLD_MULTI_THREAD_PAGE"))
-    printf("multiThreadedPageIn %ld/%ld\n", totalBytes, state->deferred.size());
-}
-#endif
+    busy->unlock();
+    delete this;
+  }
+};
 
 static void multiThreadedPageIn(DeferredFiles deferred) {
-#ifndef _WIN32
-  static pthread_t running;
-  static pthread_mutex_t busy;
+  static std::thread *running;
+  static std::mutex busy;
 
-  if (running)
-    pthread_join(running, nullptr);
-  else
-    pthread_mutex_init(&busy, nullptr);
-
-  PageInState *state =
-      new PageInState{deferred, 0, 0, llvm::sys::Process::getPageSizeEstimate(),
-                      pthread_mutex_t()};
-
-  pthread_mutex_lock(&busy);
-  pthread_create(
-      &running, nullptr,
-      [](void *ptr) -> void * {
-        PageInState *state = (PageInState *)ptr;
-        multiThreadedPageInBackground(state);
-        pthread_mutex_unlock(&busy);
-        delete state;
-        return nullptr;
-      },
-      state);
-#endif
+  busy.lock();
+  if (running) {
+    running->join();
+    delete running;
+  }
+
+  running = new std::thread(&PageInState::multiThreadedPageInBackground,
+                            new PageInState(deferred, &busy));
 }
 
 static InputFile *processFile(std::optional<MemoryBufferRef> buffer,
@@ -1432,8 +1413,7 @@ static void createFiles(const InputArgList &args) {
     }
 
     // flush threads
-    deferredFiles.clear();
-    multiThreadedPageIn(deferredFiles);
+    multiThreadedPageIn(DeferredFiles());
   }
 }
 
diff --git a/lld/MachO/Options.td b/lld/MachO/Options.td
index 3dc98fccc1b7b..2e70695868fb6 100644
--- a/lld/MachO/Options.td
+++ b/lld/MachO/Options.td
@@ -397,7 +397,7 @@ def interposable : Flag<["-"], "interposable">,
     HelpText<"Indirects access to all exported symbols in an image">,
     Group<grp_opts>;
 def read_threads : Joined<["--"], "read-threads=">,
-    HelpText<"Number of threads to use paging in files.">,
+    HelpText<"Number of threads to use if pro-actively paging in files.">,
     Group<grp_lld>;
 def order_file : Separate<["-"], "order_file">,
     MetaVarName<"<file>">,

From 817036b93dedc0b9f64ec6e6ed51a4815005b086 Mon Sep 17 00:00:00 2001
From: John Holdsworth <github@johnholdsworth.com>
Date: Sat, 12 Jul 2025 14:20:42 +0200
Subject: [PATCH 06/21] Second review.

---
 lld/MachO/Driver.cpp | 119 ++++++++++++++++++++++---------------------
 1 file changed, 60 insertions(+), 59 deletions(-)

diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index 4f3b562668ab1..6fa7a22090a91 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -288,71 +288,72 @@ class DeferredFile {
 public:
   StringRef path;
   bool isLazy;
-  std::optional<MemoryBufferRef> buffer;
-  const char *start;
-  size_t size;
+  MemoryBufferRef buffer;
 };
 using DeferredFiles = std::vector<DeferredFile>;
 
-class PageInState {
-  DeferredFiles deferred;
-  size_t counter = 0, total = 0, pageSize;
-  std::mutex mutex, *busy;
+// Most input files have been mapped but not yet paged in.
+// This code forces the page-ins on multiple threads so
+// the process is not stalled waiting on disk buffer i/o.
+void multiThreadedPageInBackground(const DeferredFiles &deferred) {
+  static size_t pageSize = Process::getPageSizeEstimate(), totalBytes;
+  size_t index = 0;
+  std::mutex mutex;
+
+  parallelFor(0, config->readThreads, [&](size_t I) {
+    while (true) {
+      mutex.lock();
+      if (index >= deferred.size()) {
+        mutex.unlock();
+        return;
+      }
+      const StringRef &buff = deferred[index].buffer.getBuffer();
+      totalBytes += buff.size();
+      index += 1;
+      mutex.unlock();
+
+      volatile int t = 0; // Reference each page to load it into memory.
+      for (const char *page = buff.data(), *end = page + buff.size();
+           page < end; page += pageSize)
+        t += *page;
+    }
+  });
 
-public:
-  PageInState(DeferredFiles &deferred, std::mutex *busy) {
-    this->deferred = deferred;
-    this->busy = busy;
-    pageSize = llvm::sys::Process::getPageSizeEstimate();
-  }
+  if (getenv("LLD_MULTI_THREAD_PAGE"))
+    llvm::dbgs() << "multiThreadedPageIn " << totalBytes << "/"
+                 << deferred.size() << "\n";
+}
+
+static void multiThreadedPageIn(const DeferredFiles &deferred) {
+  static std::thread *running;
+  static std::mutex mutex;
+  static std::deque<DeferredFiles *> queue;
 
-  // Most input files have been mapped but not yet paged in.
-  // This code forces the page-ins on multiple threads so
-  // the process is not stalled waiting on disk buffer i/o.
-  void multiThreadedPageInBackground() {
-    static size_t totalBytes;
+  mutex.lock();
+  if (running) {
+    running->join();
+    delete running;
+    running = nullptr;
+  }
 
-    parallelFor(0, config->readThreads, [&](size_t I) {
+  if (!deferred.empty()) {
+    queue.emplace_back(new DeferredFiles(deferred));
+    running = new std::thread([&]() {
       while (true) {
         mutex.lock();
-        if (counter >= deferred.size()) {
+        if (queue.empty()) {
           mutex.unlock();
           return;
         }
-        DeferredFile &file = deferred[counter];
-        totalBytes += file.size;
-        counter += 1;
+        DeferredFiles *deferred = queue.front();
+        queue.pop_front();
         mutex.unlock();
-
-        int t = 0; // Reference each page to load it into memory.
-        for (const char *page = file.start, *end = page + file.size; page < end;
-             page += pageSize)
-          t += *page;
-        total += t; // Avoids the loop being optimised out.
+        multiThreadedPageInBackground(*deferred);
+        delete deferred;
       }
     });
-
-    if (getenv("LLD_MULTI_THREAD_PAGE"))
-      llvm::dbgs() << "multiThreadedPageIn " << totalBytes << "/"
-                   << deferred.size() << "\n";
-
-    busy->unlock();
-    delete this;
   }
-};
-
-static void multiThreadedPageIn(DeferredFiles deferred) {
-  static std::thread *running;
-  static std::mutex busy;
-
-  busy.lock();
-  if (running) {
-    running->join();
-    delete running;
-  }
-
-  running = new std::thread(&PageInState::multiThreadedPageInBackground,
-                            new PageInState(deferred, &busy));
+  mutex.unlock();
 }
 
 static InputFile *processFile(std::optional<MemoryBufferRef> buffer,
@@ -454,9 +455,7 @@ static InputFile *processFile(std::optional<MemoryBufferRef> buffer,
           }
 
           if (archiveContents)
-            archiveContents->push_back({path, isLazy, std::nullopt,
-                                        mb->getBuffer().data(),
-                                        mb->getBuffer().size()});
+            archiveContents->push_back({path, isLazy, *mb});
           if (!hasObjCSection(*mb))
             continue;
           if (Error e = file->fetch(c, "-ObjC"))
@@ -530,9 +529,10 @@ static InputFile *addFile(StringRef path, LoadType loadType,
 
 static void deferFile(StringRef path, bool isLazy, DeferredFiles &deferred) {
   std::optional<MemoryBufferRef> buffer = readFile(path);
+  if (!buffer)
+    return;
   if (config->readThreads)
-    deferred.push_back({path, isLazy, buffer, buffer->getBuffer().data(),
-                        buffer->getBuffer().size()});
+    deferred.push_back({path, isLazy, *buffer});
   else
     processFile(buffer, nullptr, path, LoadType::CommandLine, isLazy);
 }
@@ -1400,11 +1400,12 @@ static void createFiles(const InputArgList &args) {
 
     DeferredFiles archiveContents;
     std::vector<ArchiveFile *> archives;
-    for (auto &file : deferredFiles)
-      if (ArchiveFile *archive = dyn_cast<ArchiveFile>(
-              processFile(file.buffer, &archiveContents, file.path,
-                          LoadType::CommandLine, file.isLazy)))
+    for (auto &file : deferredFiles) {
+      auto inputFile = processFile(file.buffer, &archiveContents, file.path,
+                                   LoadType::CommandLine, file.isLazy);
+      if (ArchiveFile *archive = dyn_cast<ArchiveFile>(inputFile))
         archives.push_back(archive);
+    }
 
     if (!archiveContents.empty()) {
       multiThreadedPageIn(archiveContents);

From c07e1686126ad0b48af10219d2787b6c23465129 Mon Sep 17 00:00:00 2001
From: John Holdsworth <github@johnholdsworth.com>
Date: Sat, 12 Jul 2025 17:18:27 +0200
Subject: [PATCH 07/21] Semms to make a difference.

---
 lld/MachO/Driver.cpp | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index 6fa7a22090a91..1e9b7cbe0c788 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -297,8 +297,8 @@ using DeferredFiles = std::vector<DeferredFile>;
 // the process is not stalled waiting on disk buffer i/o.
 void multiThreadedPageInBackground(const DeferredFiles &deferred) {
   static size_t pageSize = Process::getPageSizeEstimate(), totalBytes;
+  static std::mutex mutex;
   size_t index = 0;
-  std::mutex mutex;
 
   parallelFor(0, config->readThreads, [&](size_t I) {
     while (true) {
@@ -324,13 +324,14 @@ void multiThreadedPageInBackground(const DeferredFiles &deferred) {
                  << deferred.size() << "\n";
 }
 
-static void multiThreadedPageIn(const DeferredFiles &deferred) {
+static void
+multiThreadedPageIn(const DeferredFiles &deferred = DeferredFiles()) {
   static std::thread *running;
   static std::mutex mutex;
   static std::deque<DeferredFiles *> queue;
 
   mutex.lock();
-  if (running) {
+  if (running && (queue.empty() || deferred.empty())) {
     running->join();
     delete running;
     running = nullptr;
@@ -338,20 +339,19 @@ static void multiThreadedPageIn(const DeferredFiles &deferred) {
 
   if (!deferred.empty()) {
     queue.emplace_back(new DeferredFiles(deferred));
-    running = new std::thread([&]() {
-      while (true) {
+    if (!running)
+      running = new std::thread([&]() {
         mutex.lock();
-        if (queue.empty()) {
+        while (!queue.empty()) {
+          DeferredFiles *deferred = queue.front();
           mutex.unlock();
-          return;
+          multiThreadedPageInBackground(*deferred);
+          delete deferred;
+          mutex.lock();
+          queue.pop_front();
         }
-        DeferredFiles *deferred = queue.front();
-        queue.pop_front();
         mutex.unlock();
-        multiThreadedPageInBackground(*deferred);
-        delete deferred;
-      }
-    });
+      });
   }
   mutex.unlock();
 }
@@ -1413,8 +1413,8 @@ static void createFiles(const InputArgList &args) {
         archive->addLazySymbols();
     }
 
-    // flush threads
-    multiThreadedPageIn(DeferredFiles());
+    // reap threads
+    // multiThreadedPageIn();
   }
 }
 

From eb4827cf03ddf753487886ab0a5f55662bc8745c Mon Sep 17 00:00:00 2001
From: John Holdsworth <github@johnholdsworth.com>
Date: Tue, 15 Jul 2025 09:19:07 +0200
Subject: [PATCH 08/21] Update lld/MachO/Driver.cpp
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Daniel Rodríguez Troitiño <drodrigueztroitino@gmail.com>
---
 lld/MachO/Driver.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index 1e9b7cbe0c788..2ea5d81c5b9c6 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -312,10 +312,10 @@ void multiThreadedPageInBackground(const DeferredFiles &deferred) {
       index += 1;
       mutex.unlock();
 
-      volatile int t = 0; // Reference each page to load it into memory.
+      // Reference each page to load it into memory.
       for (const char *page = buff.data(), *end = page + buff.size();
            page < end; page += pageSize)
-        t += *page;
+        volatile char t = *page;
     }
   });
 

From ce93ae3f219aeecd2c82391cc11fa16b4553e3a4 Mon Sep 17 00:00:00 2001
From: John Holdsworth <github@johnholdsworth.com>
Date: Tue, 15 Jul 2025 17:25:40 +0200
Subject: [PATCH 09/21] De-Obfuscate loop and thread reaping.

---
 lld/MachO/Driver.cpp | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index 2ea5d81c5b9c6..4ef845ad15bf7 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -324,11 +324,10 @@ void multiThreadedPageInBackground(const DeferredFiles &deferred) {
                  << deferred.size() << "\n";
 }
 
-static void
-multiThreadedPageIn(const DeferredFiles &deferred = DeferredFiles()) {
+static void multiThreadedPageIn(const DeferredFiles &deferred) {
+  static std::deque<DeferredFiles *> queue;
   static std::thread *running;
   static std::mutex mutex;
-  static std::deque<DeferredFiles *> queue;
 
   mutex.lock();
   if (running && (queue.empty() || deferred.empty())) {
@@ -341,16 +340,18 @@ multiThreadedPageIn(const DeferredFiles &deferred = DeferredFiles()) {
     queue.emplace_back(new DeferredFiles(deferred));
     if (!running)
       running = new std::thread([&]() {
-        mutex.lock();
-        while (!queue.empty()) {
+        while (true) {
+          mutex.lock();
+          if (queue.empty()) {
+            mutex.unlock();
+            return;
+          }
           DeferredFiles *deferred = queue.front();
+          queue.pop_front();
           mutex.unlock();
           multiThreadedPageInBackground(*deferred);
           delete deferred;
-          mutex.lock();
-          queue.pop_front();
         }
-        mutex.unlock();
       });
   }
   mutex.unlock();
@@ -1413,8 +1414,8 @@ static void createFiles(const InputArgList &args) {
         archive->addLazySymbols();
     }
 
-    // reap threads
-    // multiThreadedPageIn();
+    DeferredFiles reapThreads;
+    multiThreadedPageIn(reapThreads);
   }
 }
 

From c47e5c393238ca59622dec320e5c1c357827404c Mon Sep 17 00:00:00 2001
From: John Holdsworth <github@johnholdsworth.com>
Date: Thu, 17 Jul 2025 11:20:55 +0200
Subject: [PATCH 10/21] Avoiding possible deadlock.

---
 lld/MachO/Driver.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index 4ef845ad15bf7..f0f48b0320643 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -331,7 +331,9 @@ static void multiThreadedPageIn(const DeferredFiles &deferred) {
 
   mutex.lock();
   if (running && (queue.empty() || deferred.empty())) {
+    mutex.unlock();
     running->join();
+    mutex.lock();
     delete running;
     running = nullptr;
   }

From 5caf5a62866cc4fa7262fc910e43d3bc22371c9d Mon Sep 17 00:00:00 2001
From: John Holdsworth <github@johnholdsworth.com>
Date: Thu, 17 Jul 2025 22:45:02 +0200
Subject: [PATCH 11/21] Update lld/MachO/Options.td

Co-authored-by: Ellis Hoag <ellis.sparky.hoag@gmail.com>
---
 lld/MachO/Options.td | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lld/MachO/Options.td b/lld/MachO/Options.td
index 2e70695868fb6..f2c016c8636bb 100644
--- a/lld/MachO/Options.td
+++ b/lld/MachO/Options.td
@@ -397,7 +397,7 @@ def interposable : Flag<["-"], "interposable">,
     HelpText<"Indirects access to all exported symbols in an image">,
     Group<grp_opts>;
 def read_threads : Joined<["--"], "read-threads=">,
-    HelpText<"Number of threads to use if pro-actively paging in files.">,
+    HelpText<"Number of threads to use if proactively paging in files.">,
     Group<grp_lld>;
 def order_file : Separate<["-"], "order_file">,
     MetaVarName<"<file>">,

From 890c492d12f407be234b6a17c3195beee9e6ab6d Mon Sep 17 00:00:00 2001
From: John Holdsworth <github@johnholdsworth.com>
Date: Thu, 17 Jul 2025 22:45:33 +0200
Subject: [PATCH 12/21] Update lld/MachO/Driver.cpp

Co-authored-by: Ellis Hoag <ellis.sparky.hoag@gmail.com>
---
 lld/MachO/Driver.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index f0f48b0320643..3f6fb9f88d597 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -296,7 +296,8 @@ using DeferredFiles = std::vector<DeferredFile>;
 // This code forces the page-ins on multiple threads so
 // the process is not stalled waiting on disk buffer i/o.
 void multiThreadedPageInBackground(const DeferredFiles &deferred) {
-  static size_t pageSize = Process::getPageSizeEstimate(), totalBytes;
+  static const size_t pageSize = Process::getPageSizeEstimate();
+  size_t totalBytes = 0;
   static std::mutex mutex;
   size_t index = 0;
 

From 9714785423ef21ba53e26cd9e879358c66afa675 Mon Sep 17 00:00:00 2001
From: John Holdsworth <github@johnholdsworth.com>
Date: Thu, 17 Jul 2025 22:45:52 +0200
Subject: [PATCH 13/21] Update lld/MachO/Driver.cpp

Co-authored-by: Ellis Hoag <ellis.sparky.hoag@gmail.com>
---
 lld/MachO/Driver.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index 3f6fb9f88d597..5d60af3625edd 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -536,7 +536,7 @@ static void deferFile(StringRef path, bool isLazy, DeferredFiles &deferred) {
   if (!buffer)
     return;
   if (config->readThreads)
-    deferred.push_back({path, isLazy, *buffer});
+    deferred.emplace_back(path, isLazy, *buffer);
   else
     processFile(buffer, nullptr, path, LoadType::CommandLine, isLazy);
 }

From 85fd77ff34b07be4ac2ff4d86453f862fdf7c4d0 Mon Sep 17 00:00:00 2001
From: John Holdsworth <github@johnholdsworth.com>
Date: Thu, 17 Jul 2025 22:46:11 +0200
Subject: [PATCH 14/21] Update lld/MachO/Driver.cpp

Co-authored-by: Ellis Hoag <ellis.sparky.hoag@gmail.com>
---
 lld/MachO/Driver.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index 5d60af3625edd..e42eec6ed73cc 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -316,7 +316,7 @@ void multiThreadedPageInBackground(const DeferredFiles &deferred) {
       // Reference each page to load it into memory.
       for (const char *page = buff.data(), *end = page + buff.size();
            page < end; page += pageSize)
-        volatile char t = *page;
+        [[maybe_unused]] volatile char t = *page;
     }
   });
 

From 6f5f7cbcb0c453f6ef31d0b76fb75a1327204862 Mon Sep 17 00:00:00 2001
From: John Holdsworth <github@johnholdsworth.com>
Date: Thu, 17 Jul 2025 23:29:30 +0200
Subject: [PATCH 15/21] Fourth review.

---
 lld/MachO/Driver.cpp | 42 ++++++++++++++++++------------------------
 lld/MachO/Options.td |  2 +-
 2 files changed, 19 insertions(+), 25 deletions(-)

diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index e42eec6ed73cc..31b14e4cba6aa 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -286,6 +286,8 @@ static void saveThinArchiveToRepro(ArchiveFile const *file) {
 
 class DeferredFile {
 public:
+  DeferredFile(StringRef path, bool isLazy, MemoryBufferRef buffer)
+      : path(path), isLazy(isLazy), buffer(buffer) {}
   StringRef path;
   bool isLazy;
   MemoryBufferRef buffer;
@@ -297,36 +299,31 @@ using DeferredFiles = std::vector<DeferredFile>;
 // the process is not stalled waiting on disk buffer i/o.
 void multiThreadedPageInBackground(const DeferredFiles &deferred) {
   static const size_t pageSize = Process::getPageSizeEstimate();
-  size_t totalBytes = 0;
-  static std::mutex mutex;
-  size_t index = 0;
+  static size_t totalBytes = 0;
+  std::atomic_int index = 0;
 
   parallelFor(0, config->readThreads, [&](size_t I) {
-    while (true) {
-      mutex.lock();
-      if (index >= deferred.size()) {
-        mutex.unlock();
-        return;
-      }
+    while (index < (int)deferred.size()) {
       const StringRef &buff = deferred[index].buffer.getBuffer();
       totalBytes += buff.size();
       index += 1;
-      mutex.unlock();
 
-      // Reference each page to load it into memory.
+      // Reference all file's mmap'd pages to load them into memory.
       for (const char *page = buff.data(), *end = page + buff.size();
            page < end; page += pageSize)
-        [[maybe_unused]] volatile char t = *page;
+        volatile char t = *page;
     }
   });
 
+#ifndef NDEBUG
   if (getenv("LLD_MULTI_THREAD_PAGE"))
     llvm::dbgs() << "multiThreadedPageIn " << totalBytes << "/"
                  << deferred.size() << "\n";
+#endif
 }
 
 static void multiThreadedPageIn(const DeferredFiles &deferred) {
-  static std::deque<DeferredFiles *> queue;
+  static std::deque<DeferredFiles> queue;
   static std::thread *running;
   static std::mutex mutex;
 
@@ -340,21 +337,18 @@ static void multiThreadedPageIn(const DeferredFiles &deferred) {
   }
 
   if (!deferred.empty()) {
-    queue.emplace_back(new DeferredFiles(deferred));
+    queue.emplace_back(deferred);
     if (!running)
       running = new std::thread([&]() {
-        while (true) {
+        mutex.lock();
+        while (!queue.empty()) {
+          const DeferredFiles &deferred = queue.front();
+          mutex.unlock();
+          multiThreadedPageInBackground(deferred);
           mutex.lock();
-          if (queue.empty()) {
-            mutex.unlock();
-            return;
-          }
-          DeferredFiles *deferred = queue.front();
           queue.pop_front();
-          mutex.unlock();
-          multiThreadedPageInBackground(*deferred);
-          delete deferred;
         }
+        mutex.unlock();
       });
   }
   mutex.unlock();
@@ -459,7 +453,7 @@ static InputFile *processFile(std::optional<MemoryBufferRef> buffer,
           }
 
           if (archiveContents)
-            archiveContents->push_back({path, isLazy, *mb});
+            archiveContents->emplace_back(path, isLazy, *mb);
           if (!hasObjCSection(*mb))
             continue;
           if (Error e = file->fetch(c, "-ObjC"))
diff --git a/lld/MachO/Options.td b/lld/MachO/Options.td
index f2c016c8636bb..dcfdbdb0038c4 100644
--- a/lld/MachO/Options.td
+++ b/lld/MachO/Options.td
@@ -397,7 +397,7 @@ def interposable : Flag<["-"], "interposable">,
     HelpText<"Indirects access to all exported symbols in an image">,
     Group<grp_opts>;
 def read_threads : Joined<["--"], "read-threads=">,
-    HelpText<"Number of threads to use if proactively paging in files.">,
+    HelpText<"Number of threads to use to proactively page in files. 20 would be a typical value, use 0 to disable this feature.">,
     Group<grp_lld>;
 def order_file : Separate<["-"], "order_file">,
     MetaVarName<"<file>">,

From e3e036974b34fa006e45533b2515a817b6271f78 Mon Sep 17 00:00:00 2001
From: John Holdsworth <github@johnholdsworth.com>
Date: Fri, 18 Jul 2025 00:13:36 +0200
Subject: [PATCH 16/21] Switch to std::atomic_int.

---
 lld/MachO/Driver.cpp | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index 31b14e4cba6aa..40eed8891d64f 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -303,10 +303,12 @@ void multiThreadedPageInBackground(const DeferredFiles &deferred) {
   std::atomic_int index = 0;
 
   parallelFor(0, config->readThreads, [&](size_t I) {
-    while (index < (int)deferred.size()) {
-      const StringRef &buff = deferred[index].buffer.getBuffer();
+    while (true) {
+      int localIndex = index.fetch_add(1);
+      if (localIndex >= (int)deferred.size())
+        break;
+      const StringRef &buff = deferred[localIndex].buffer.getBuffer();
       totalBytes += buff.size();
-      index += 1;
 
       // Reference all file's mmap'd pages to load them into memory.
       for (const char *page = buff.data(), *end = page + buff.size();

From febf5a9158cd1a3f9c4c0f345cf9503c893ecb58 Mon Sep 17 00:00:00 2001
From: John Holdsworth <github@johnholdsworth.com>
Date: Fri, 18 Jul 2025 00:50:55 +0200
Subject: [PATCH 17/21] Switch to std::unique_ptr.

---
 lld/MachO/Driver.cpp | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index 40eed8891d64f..c331966551f0c 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -325,7 +325,7 @@ void multiThreadedPageInBackground(const DeferredFiles &deferred) {
 }
 
 static void multiThreadedPageIn(const DeferredFiles &deferred) {
-  static std::deque<DeferredFiles> queue;
+  static std::deque<std::unique_ptr<DeferredFiles>> queue;
   static std::thread *running;
   static std::mutex mutex;
 
@@ -339,12 +339,13 @@ static void multiThreadedPageIn(const DeferredFiles &deferred) {
   }
 
   if (!deferred.empty()) {
-    queue.emplace_back(deferred);
+    queue.emplace_back(
+        std::unique_ptr<DeferredFiles>(new DeferredFiles(deferred)));
     if (!running)
       running = new std::thread([&]() {
         mutex.lock();
         while (!queue.empty()) {
-          const DeferredFiles &deferred = queue.front();
+          const DeferredFiles &deferred = *queue.front();
           mutex.unlock();
           multiThreadedPageInBackground(deferred);
           mutex.lock();

From a5f7a42ca3ff8ec842dfd9bbc656403119f48a0b Mon Sep 17 00:00:00 2001
From: John Holdsworth <github@johnholdsworth.com>
Date: Fri, 18 Jul 2025 11:41:33 +0200
Subject: [PATCH 18/21] Remove a couple of warnings.

---
 lld/MachO/Driver.cpp | 5 ++---
 lld/MachO/Options.td | 2 +-
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index c331966551f0c..ebf1b4e97bdb7 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -311,17 +311,16 @@ void multiThreadedPageInBackground(const DeferredFiles &deferred) {
       totalBytes += buff.size();
 
       // Reference all file's mmap'd pages to load them into memory.
+      volatile char t = 0;
       for (const char *page = buff.data(), *end = page + buff.size();
            page < end; page += pageSize)
-        volatile char t = *page;
+        t += *page;
     }
   });
 
-#ifndef NDEBUG
   if (getenv("LLD_MULTI_THREAD_PAGE"))
     llvm::dbgs() << "multiThreadedPageIn " << totalBytes << "/"
                  << deferred.size() << "\n";
-#endif
 }
 
 static void multiThreadedPageIn(const DeferredFiles &deferred) {
diff --git a/lld/MachO/Options.td b/lld/MachO/Options.td
index dcfdbdb0038c4..57425824dd627 100644
--- a/lld/MachO/Options.td
+++ b/lld/MachO/Options.td
@@ -397,7 +397,7 @@ def interposable : Flag<["-"], "interposable">,
     HelpText<"Indirects access to all exported symbols in an image">,
     Group<grp_opts>;
 def read_threads : Joined<["--"], "read-threads=">,
-    HelpText<"Number of threads to use to proactively page in files. 20 would be a typical value, use 0 to disable this feature.">,
+    HelpText<"Number of threads to use to proactively page in files for faster disk i/o. 20 would be a typical value, use 0 to disable this feature.">,
     Group<grp_lld>;
 def order_file : Separate<["-"], "order_file">,
     MetaVarName<"<file>">,

From 6b874b2bf6c6ee571d5d283ba36f7f884688640f Mon Sep 17 00:00:00 2001
From: John Holdsworth <github@johnholdsworth.com>
Date: Fri, 18 Jul 2025 21:02:04 +0200
Subject: [PATCH 19/21] Try LLVM_ATTRIBUTE_UNUSED

---
 lld/MachO/Driver.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index ebf1b4e97bdb7..91758f3e8a51f 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -311,10 +311,9 @@ void multiThreadedPageInBackground(const DeferredFiles &deferred) {
       totalBytes += buff.size();
 
       // Reference all file's mmap'd pages to load them into memory.
-      volatile char t = 0;
       for (const char *page = buff.data(), *end = page + buff.size();
            page < end; page += pageSize)
-        t += *page;
+        LLVM_ATTRIBUTE_UNUSED volatile char t = *page;
     }
   });
 

From 84154d43445fa410e99ea5d8f3b7300cf7780f12 Mon Sep 17 00:00:00 2001
From: John Holdsworth <github@johnholdsworth.com>
Date: Fri, 18 Jul 2025 22:12:22 +0200
Subject: [PATCH 20/21] Update lld/MachO/Driver.cpp
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Daniel Rodríguez Troitiño <drodrigueztroitino@gmail.com>
---
 lld/MachO/Driver.cpp | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index 91758f3e8a51f..85f357bb55d2c 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -341,15 +341,17 @@ static void multiThreadedPageIn(const DeferredFiles &deferred) {
         std::unique_ptr<DeferredFiles>(new DeferredFiles(deferred)));
     if (!running)
       running = new std::thread([&]() {
-        mutex.lock();
-        while (!queue.empty()) {
-          const DeferredFiles &deferred = *queue.front();
-          mutex.unlock();
-          multiThreadedPageInBackground(deferred);
-          mutex.lock();
-          queue.pop_front();
+        while (true) {
+           mutex.lock();
+           if (queue.empty) {
+             mutex.unlock();
+             break;
+           }
+           DeferredFiles deferred(*queue.front());
+           queue.pop_front();
+           mutex.unlock();
+           multiThreadedPageInBackground(deferred);
         }
-        mutex.unlock();
       });
   }
   mutex.unlock();

From ed9f07e8c26d876295388c41c34177619fc40d20 Mon Sep 17 00:00:00 2001
From: John Holdsworth <github@johnholdsworth.com>
Date: Fri, 18 Jul 2025 23:44:36 +0200
Subject: [PATCH 21/21] Comparing inner loops.

---
 lld/MachO/Driver.cpp | 43 ++++++++++++++++++++++++++++++-------------
 1 file changed, 30 insertions(+), 13 deletions(-)

diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index 85f357bb55d2c..357ef58e524e1 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -31,6 +31,7 @@
 #include "lld/Common/Reproduce.h"
 #include "lld/Common/Version.h"
 #include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/BinaryFormat/MachO.h"
@@ -41,6 +42,7 @@
 #include "llvm/Object/Archive.h"
 #include "llvm/Option/ArgList.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Parallel.h"
 #include "llvm/Support/Path.h"
@@ -297,8 +299,24 @@ using DeferredFiles = std::vector<DeferredFile>;
 // Most input files have been mapped but not yet paged in.
 // This code forces the page-ins on multiple threads so
 // the process is not stalled waiting on disk buffer i/o.
-void multiThreadedPageInBackground(const DeferredFiles &deferred) {
+void multiThreadedPageInBackground(DeferredFiles &deferred) {
   static const size_t pageSize = Process::getPageSizeEstimate();
+#if 0
+  ThreadPoolStrategy oldStrategy = llvm::parallel::strategy;
+  (void)llvm::make_scope_exit([&]() { llvm::parallel::strategy = oldStrategy; });
+  llvm::parallel::strategy = llvm::hardware_concurrency(config->readThreads);
+
+  size_t totalBytes = parallelTransformReduce(deferred, 0,
+    [](size_t acc, size_t size) { return acc + size; },
+    [&](DeferredFile &file) {
+      const StringRef &buffer = file.buffer.getBuffer();
+      for (const char *page = buffer.data(), *end = page + buffer.size();
+           page < end; page += pageSize)
+        LLVM_ATTRIBUTE_UNUSED volatile char t = *page;
+      return buffer.size();
+    }
+    );
+#else
   static size_t totalBytes = 0;
   std::atomic_int index = 0;
 
@@ -316,7 +334,7 @@ void multiThreadedPageInBackground(const DeferredFiles &deferred) {
         LLVM_ATTRIBUTE_UNUSED volatile char t = *page;
     }
   });
-
+#endif
   if (getenv("LLD_MULTI_THREAD_PAGE"))
     llvm::dbgs() << "multiThreadedPageIn " << totalBytes << "/"
                  << deferred.size() << "\n";
@@ -337,20 +355,19 @@ static void multiThreadedPageIn(const DeferredFiles &deferred) {
   }
 
   if (!deferred.empty()) {
-    queue.emplace_back(
-        std::unique_ptr<DeferredFiles>(new DeferredFiles(deferred)));
+    queue.emplace_back(std::make_unique<DeferredFiles>(deferred));
     if (!running)
       running = new std::thread([&]() {
         while (true) {
-           mutex.lock();
-           if (queue.empty) {
-             mutex.unlock();
-             break;
-           }
-           DeferredFiles deferred(*queue.front());
-           queue.pop_front();
-           mutex.unlock();
-           multiThreadedPageInBackground(deferred);
+          mutex.lock();
+          if (queue.empty()) {
+            mutex.unlock();
+            break;
+          }
+          DeferredFiles deferred(*queue.front());
+          queue.pop_front();
+          mutex.unlock();
+          multiThreadedPageInBackground(deferred);
         }
       });
   }