diff --git a/src/openrct2/core/FileIndex.hpp b/src/openrct2/core/FileIndex.hpp index 4cca33de24..e2440dda8a 100644 --- a/src/openrct2/core/FileIndex.hpp +++ b/src/openrct2/core/FileIndex.hpp @@ -20,6 +20,8 @@ #include #include #include +#include +#include #include "../common.h" #include "Console.hpp" #include "File.h" @@ -178,32 +180,85 @@ private: return ScanResult(stats, files); } - std::vector Build(const ScanResult &scanResult) const + void BuildRange(const ScanResult &scanResult, + size_t rangeStart, + size_t rangeEnd, + std::vector& items, + std::mutex& printLock) const { - std::vector items; - Console::WriteLine("Building %s (%zu items)", _name.c_str(), scanResult.Files.size()); - - auto startTime = std::chrono::high_resolution_clock::now(); - // Start at 1, so that we can reach 100% completion status - size_t i = 1; - for (auto filePath : scanResult.Files) + items.reserve(rangeEnd - rangeStart); + for (size_t i = rangeStart; i < rangeEnd; i++) { - Console::WriteFormat("File %5d of %d, done %3d%%\r", i, scanResult.Files.size(), i * 100 / scanResult.Files.size()); - i++; - log_verbose("FileIndex:Indexing '%s'", filePath.c_str()); + const auto& filePath = scanResult.Files.at(i); + + if (_log_levels[DIAGNOSTIC_LEVEL_VERBOSE]) + { + std::unique_lock lock(printLock); + log_verbose("FileIndex:Indexing '%s'", filePath.c_str()); + } + auto item = Create(filePath); if (std::get<0>(item)) { items.push_back(std::get<1>(item)); } } + } - WriteIndexFile(scanResult.Stats, items); + std::vector Build(const ScanResult &scanResult) const + { + std::vector allItems; + Console::WriteLine("Building %s (%zu items)", _name.c_str(), scanResult.Files.size()); + + auto startTime = std::chrono::high_resolution_clock::now(); + + const size_t totalCount = scanResult.Files.size(); + if (totalCount > 0) + { + const size_t numThreads = std::thread::hardware_concurrency(); + + size_t stepSize = totalCount / numThreads; + + std::vector threads; + std::vector> containers; + std::mutex printLock; // For verbose prints. + containers.resize(numThreads + (totalCount % stepSize == 0 ? 0 : 1)); + + for (size_t rangeStart = 0; rangeStart < totalCount; rangeStart += stepSize) + { + if (rangeStart + stepSize > totalCount) + stepSize = totalCount - rangeStart; + + auto& items = containers[threads.size()]; + + threads.emplace_back(&FileIndex::BuildRange, + this, + std::cref(scanResult), + rangeStart, + rangeStart + stepSize, + std::ref(items), + std::ref(printLock)); + } + + for (auto&& itr : threads) + { + if (itr.joinable()) + itr.join(); + } + + for (auto&& itr : containers) + { + allItems.insert(allItems.end(), itr.begin(), itr.end()); + } + + WriteIndexFile(scanResult.Stats, allItems); + } auto endTime = std::chrono::high_resolution_clock::now(); auto duration = (std::chrono::duration)(endTime - startTime); Console::WriteLine("Finished building %s in %.2f seconds.", _name.c_str(), duration.count()); - return items; + + return allItems; } std::tuple> ReadIndexFile(const DirectoryStats &stats) const @@ -229,6 +284,7 @@ private: header.Stats.FileDateModifiedChecksum == stats.FileDateModifiedChecksum && header.Stats.PathChecksum == stats.PathChecksum) { + items.reserve(header.NumItems); // Directory is the same, just read the saved items for (uint32 i = 0; i < header.NumItems; i++) { @@ -258,7 +314,7 @@ private: log_verbose("FileIndex:Writing index: '%s'", _indexPath.c_str()); Path::CreateDirectory(Path::GetDirectory(_indexPath)); auto fs = FileStream(_indexPath, FILE_MODE_WRITE); - + // Write header FileIndexHeader header; header.MagicNumber = _magicNumber; @@ -268,7 +324,7 @@ private: header.Stats = stats; header.NumItems = (uint32)items.size(); fs.WriteValue(header); - + // Write items for (const auto item : items) {