Use multiple threads to generate object index cache.

src/openrct2/core/FileIndex.hpp

@@ -20,6 +20,8 @@
 #include <string>
 #include <tuple>
 #include <vector>
+#include <thread>
+#include <mutex>
 #include "../common.h"
 #include "Console.hpp"
 #include "File.h"
@@ -178,32 +180,85 @@ private:
         return ScanResult(stats, files);
     }
 
-    std::vector<TItem> Build(const ScanResult &scanResult) const
+    void BuildRange(const ScanResult &scanResult,
+                    size_t rangeStart,
+                    size_t rangeEnd,
+                    std::vector<TItem>& items,
+                    std::mutex& printLock) const
     {
-        std::vector<TItem> items;
-        Console::WriteLine("Building %s (%zu items)", _name.c_str(), scanResult.Files.size());
-
-        auto startTime = std::chrono::high_resolution_clock::now();
-        // Start at 1, so that we can reach 100% completion status
-        size_t i = 1;
-        for (auto filePath : scanResult.Files)
+        items.reserve(rangeEnd - rangeStart);
+        for (size_t i = rangeStart; i < rangeEnd; i++)
         {
-            Console::WriteFormat("File %5d of %d, done %3d%%\r", i, scanResult.Files.size(), i * 100 / scanResult.Files.size());
-            i++;
-            log_verbose("FileIndex:Indexing '%s'", filePath.c_str());
+            const auto& filePath = scanResult.Files.at(i);
+
+            if (_log_levels[DIAGNOSTIC_LEVEL_VERBOSE])
+            {
+                std::unique_lock<std::mutex> lock(printLock);
+                log_verbose("FileIndex:Indexing '%s'", filePath.c_str());
+            }
+
             auto item = Create(filePath);
             if (std::get<0>(item))
             {
                 items.push_back(std::get<1>(item));
             }
         }
+    }
 
-        WriteIndexFile(scanResult.Stats, items);
+    std::vector<TItem> Build(const ScanResult &scanResult) const
+    {
+        std::vector<TItem> allItems;
+        Console::WriteLine("Building %s (%zu items)", _name.c_str(), scanResult.Files.size());
+
+        auto startTime = std::chrono::high_resolution_clock::now();
+
+        const size_t totalCount = scanResult.Files.size();
+        if (totalCount > 0)
+        {
+            const size_t numThreads = std::thread::hardware_concurrency();
+
+            size_t stepSize = totalCount / numThreads;
+
+            std::vector<std::thread> threads;
+            std::vector<std::vector<TItem>> containers;
+            std::mutex printLock; // For verbose prints.
+            containers.resize(numThreads + (totalCount % stepSize == 0 ? 0 : 1));
+
+            for (size_t rangeStart = 0; rangeStart < totalCount; rangeStart += stepSize)
+            {
+                if (rangeStart + stepSize > totalCount)
+                    stepSize = totalCount - rangeStart;
+
+                auto& items = containers[threads.size()];
+
+                threads.emplace_back(&FileIndex<TItem>::BuildRange,
+                    this,
+                    std::cref(scanResult),
+                    rangeStart,
+                    rangeStart + stepSize,
+                    std::ref(items),
+                    std::ref(printLock));
+            }
+
+            for (auto&& itr : threads)
+            {
+                if (itr.joinable())
+                    itr.join();
+            }
+
+            for (auto&& itr : containers)
+            {
+                allItems.insert(allItems.end(), itr.begin(), itr.end());
+            }
+
+            WriteIndexFile(scanResult.Stats, allItems);
+        }
 
         auto endTime = std::chrono::high_resolution_clock::now();
         auto duration = (std::chrono::duration<float>)(endTime - startTime);
         Console::WriteLine("Finished building %s in %.2f seconds.", _name.c_str(), duration.count());
-        return items;
+
+        return allItems;
     }
 
     std::tuple<bool, std::vector<TItem>> ReadIndexFile(const DirectoryStats &stats) const
@@ -229,6 +284,7 @@ private:
                     header.Stats.FileDateModifiedChecksum == stats.FileDateModifiedChecksum &&
                     header.Stats.PathChecksum == stats.PathChecksum)
                 {
+                    items.reserve(header.NumItems);
                     // Directory is the same, just read the saved items
                     for (uint32 i = 0; i < header.NumItems; i++)
                     {
@@ -258,7 +314,7 @@ private:
             log_verbose("FileIndex:Writing index: '%s'", _indexPath.c_str());
             Path::CreateDirectory(Path::GetDirectory(_indexPath));
             auto fs = FileStream(_indexPath, FILE_MODE_WRITE);
-    
+
             // Write header
             FileIndexHeader header;
             header.MagicNumber = _magicNumber;
@@ -268,7 +324,7 @@ private:
             header.Stats = stats;
             header.NumItems = (uint32)items.size();
             fs.WriteValue(header);
-    
+
             // Write items
             for (const auto item : items)
             {