mirrors
/
OpenRCT2
mirror of https://github.com/OpenRCT2/OpenRCT2.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #7435 from ZehMatt/indexing-perf 

Use multiple threads to generate object index cache.
This commit is contained in:
Michael Steenbeek 2018-04-30 14:55:54 +02:00 committed by GitHub
parent 2fc01cb698 9caa30f846
commit 3025599eb3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 339 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
OpenRCT2.xcodeproj/project.pbxproj
View File

@ -3487,7 +3487,7 @@
isa = XCBuildConfiguration;
buildSettings = {
ALWAYS_SEARCH_USER_PATHS = NO;
CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
CLANG_CXX_LANGUAGE_STANDARD = "gnu++17";
CLANG_CXX_LIBRARY = "libc++";
CLANG_ENABLE_MODULES = YES;
CLANG_ENABLE_OBJC_ARC = NO;
@ -3535,7 +3535,7 @@
isa = XCBuildConfiguration;
buildSettings = {
ALWAYS_SEARCH_USER_PATHS = NO;
CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
CLANG_CXX_LANGUAGE_STANDARD = "gnu++17";
CLANG_CXX_LIBRARY = "libc++";
CLANG_ENABLE_MODULES = YES;
CLANG_ENABLE_OBJC_ARC = NO;
@ -3581,7 +3581,7 @@
buildSettings = {
ALWAYS_SEARCH_USER_PATHS = NO;
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
CLANG_CXX_LANGUAGE_STANDARD = "gnu++17";
CLANG_CXX_LIBRARY = "libc++";
CLANG_WARN_UNREACHABLE_CODE = NO;
CLANG_X86_VECTOR_INSTRUCTIONS = default;
@ -3618,7 +3618,7 @@
buildSettings = {
ALWAYS_SEARCH_USER_PATHS = NO;
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
CLANG_CXX_LANGUAGE_STANDARD = "gnu++17";
CLANG_CXX_LIBRARY = "libc++";
CLANG_WARN_UNREACHABLE_CODE = NO;
CLANG_X86_VECTOR_INSTRUCTIONS = default;
@ -3655,7 +3655,7 @@
buildSettings = {
CLANG_ANALYZER_NONNULL = YES;
CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
CLANG_CXX_LANGUAGE_STANDARD = "gnu++17";
CLANG_ENABLE_OBJC_ARC = NO;
CLANG_WARN_DOCUMENTATION_COMMENTS = NO;
CLANG_WARN_UNREACHABLE_CODE = NO;
@ -3693,7 +3693,7 @@
buildSettings = {
CLANG_ANALYZER_NONNULL = YES;
CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
CLANG_CXX_LANGUAGE_STANDARD = "gnu++17";
CLANG_ENABLE_OBJC_ARC = NO;
CLANG_WARN_DOCUMENTATION_COMMENTS = NO;
CLANG_WARN_UNREACHABLE_CODE = NO;
@ -3731,7 +3731,7 @@
buildSettings = {
CLANG_ANALYZER_NONNULL = YES;
CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
CLANG_CXX_LANGUAGE_STANDARD = "gnu++17";
CLANG_ENABLE_OBJC_ARC = NO;
CLANG_WARN_DOCUMENTATION_COMMENTS = NO;
CODE_SIGN_IDENTITY = "";
@ -3763,7 +3763,7 @@
buildSettings = {
CLANG_ANALYZER_NONNULL = YES;
CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
CLANG_CXX_LANGUAGE_STANDARD = "gnu++14";
CLANG_CXX_LANGUAGE_STANDARD = "gnu++17";
CLANG_ENABLE_OBJC_ARC = NO;
CLANG_WARN_DOCUMENTATION_COMMENTS = NO;
CODE_SIGN_IDENTITY = "";

3
distribution/changelog.txt
View File

@ -21,7 +21,8 @@
- Fix: Cut-away view does not draw tile elements that have been moved down on the list.
- Improved: [#2989] Multiplayer window now changes title when tab changes.
- Improved: [#5339] Change eyedropper icon to actual eyedropper and change cursor to crosshair.
- Improved: Raising land near the map edge makes the affected area smaller instead of showing an 'off edge map' error.
- Improved: [#7302] Raising land near the map edge makes the affected area smaller instead of showing an 'off edge map' error.
- Improved: [#7435] Object indexing now supports multi-threading.
0.1.2 (2018-03-18)
------------------------------------------------------------------------

62
src/openrct2/Diagnostic.cpp
View File

@ -16,6 +16,7 @@
#include <cstdarg>
#include <cstdio>
#include "core/String.hpp"
#include "Diagnostic.h"
#ifdef __ANDROID__
@ -80,46 +81,45 @@ static constexpr const char * _level_strings[] = {
void diagnostic_log(DIAGNOSTIC_LEVEL diagnosticLevel, const char *format, ...)
{
va_list args;
if (_log_levels[diagnosticLevel])
{
// Level
auto prefix = String::StdFormat("%s: ", _level_strings[diagnosticLevel]);
if (!_log_levels[diagnosticLevel])
return;
// Message
va_start(args, format);
auto msg = String::StdFormat_VA(format, args);
va_end(args);
FILE * stream = diagnostic_get_stream(diagnosticLevel);
// Level
fprintf(stream, "%s: ", _level_strings[diagnosticLevel]);
// Message
va_start(args, format);
vfprintf(stream, format, args);
va_end(args);
// Line terminator
fprintf(stream, "\n");
auto stream = diagnostic_get_stream(diagnosticLevel);
fprintf(stream, "%s%s\n", prefix.c_str(), msg.c_str());
}
}
void diagnostic_log_with_location(DIAGNOSTIC_LEVEL diagnosticLevel, const char *file, const char *function, sint32 line, const char *format, ...)
{
va_list args;
if (_log_levels[diagnosticLevel])
{
// Level and source code information
std::string prefix;
if (_log_location_enabled)
{
prefix = String::StdFormat("%s[%s:%d (%s)]: ", _level_strings[diagnosticLevel], file, line, function);
}
else
{
prefix = String::StdFormat("%s: ", _level_strings[diagnosticLevel]);
}
if (!_log_levels[diagnosticLevel])
return;
// Message
va_start(args, format);
auto msg = String::StdFormat_VA(format, args);
va_end(args);
FILE * stream = diagnostic_get_stream(diagnosticLevel);
// Level and source code information
if (_log_location_enabled)
fprintf(stream, "%s[%s:%d (%s)]: ", _level_strings[diagnosticLevel], file, line, function);
else
fprintf(stream, "%s: ", _level_strings[diagnosticLevel]);
// Message
va_start(args, format);
vfprintf(stream, format, args);
va_end(args);
// Line terminator
fprintf(stream, "\n");
auto stream = diagnostic_get_stream(diagnosticLevel);
fprintf(stream, "%s%s\n", prefix.c_str(), msg.c_str());
}
}
#endif // __ANDROID__

15
src/openrct2/core/Console.cpp
View File

@ -15,6 +15,7 @@
#pragma endregion
#include <cstdio>
#include <string>
#include "Console.hpp"
#include "../platform/platform.h"
@ -33,10 +34,8 @@ namespace Console
void WriteSpace(size_t count)
{
for (size_t i = 0; i < count; i++)
{
Write(' ');
}
std::string sz(count, ' ');
Write(sz.c_str());
}
void WriteFormat(const utf8 * format, ...)
@ -58,8 +57,8 @@ namespace Console
va_list args;
va_start(args, format);
vfprintf(stdout, format, args);
puts("");
auto formatLn = std::string(format) + "\n";
vfprintf(stdout, formatLn.c_str(), args);
va_end(args);
}
@ -99,8 +98,8 @@ namespace Console
void WriteLine_VA(const utf8 * format, va_list args)
{
vfprintf(stdout, format, args);
puts("");
auto formatLn = std::string(format) + "\n";
vfprintf(stdout, formatLn.c_str(), args);
}
}
}

99
src/openrct2/core/FileIndex.hpp
View File

@ -20,11 +20,13 @@
#include <string>
#include <tuple>
#include <vector>
#include <list>
#include "../common.h"
#include "Console.hpp"
#include "File.h"
#include "FileScanner.h"
#include "FileStream.hpp"
#include "JobPool.hpp"
#include "Path.hpp"
template<typename TItem>
@ -178,32 +180,96 @@ private:
return ScanResult(stats, files);
}
std::vector<TItem> Build(const ScanResult &scanResult) const
void BuildRange(const ScanResult &scanResult,
size_t rangeStart,
size_t rangeEnd,
std::vector<TItem>& items,
std::atomic<size_t>& processed,
std::mutex& printLock) const
{
std::vector<TItem> items;
Console::WriteLine("Building %s (%zu items)", _name.c_str(), scanResult.Files.size());
auto startTime = std::chrono::high_resolution_clock::now();
// Start at 1, so that we can reach 100% completion status
size_t i = 1;
for (auto filePath : scanResult.Files)
items.reserve(rangeEnd - rangeStart);
for (size_t i = rangeStart; i < rangeEnd; i++)
{
Console::WriteFormat("File %5d of %d, done %3d%%\r", i, scanResult.Files.size(), i * 100 / scanResult.Files.size());
i++;
log_verbose("FileIndex:Indexing '%s'", filePath.c_str());
const auto& filePath = scanResult.Files.at(i);
if (_log_levels[DIAGNOSTIC_LEVEL_VERBOSE])
{
std::lock_guard<std::mutex> lock(printLock);
log_verbose("FileIndex:Indexing '%s'", filePath.c_str());
}
auto item = Create(filePath);
if (std::get<0>(item))
{
items.push_back(std::get<1>(item));
}
}
WriteIndexFile(scanResult.Stats, items);
processed++;
}
}
std::vector<TItem> Build(const ScanResult &scanResult) const
{
std::vector<TItem> allItems;
Console::WriteLine("Building %s (%zu items)", _name.c_str(), scanResult.Files.size());
auto startTime = std::chrono::high_resolution_clock::now();
const size_t totalCount = scanResult.Files.size();
if (totalCount > 0)
{
JobPool jobPool;
std::mutex printLock; // For verbose prints.
std::list<std::vector<TItem>> containers;
size_t stepSize = 100; // Handpicked, seems to work well with 4/8 cores.
std::atomic<size_t> processed = ATOMIC_VAR_INIT(0);
auto reportProgress =
[&]()
{
const size_t completed = processed;
Console::WriteFormat("File %5d of %d, done %3d%%\r", completed, totalCount, completed * 100 / totalCount);
};
for (size_t rangeStart = 0; rangeStart < totalCount; rangeStart += stepSize)
{
if (rangeStart + stepSize > totalCount)
{
stepSize = totalCount - rangeStart;
}
auto& items = containers.emplace_back();
jobPool.AddTask(std::bind(&FileIndex<TItem>::BuildRange,
this,
std::cref(scanResult),
rangeStart,
rangeStart + stepSize,
std::ref(items),
std::ref(processed),
std::ref(printLock)));
reportProgress();
}
jobPool.Join(reportProgress);
for (auto&& itr : containers)
{
allItems.insert(allItems.end(), itr.begin(), itr.end());
}
WriteIndexFile(scanResult.Stats, allItems);
}
auto endTime = std::chrono::high_resolution_clock::now();
auto duration = (std::chrono::duration<float>)(endTime - startTime);
Console::WriteLine("Finished building %s in %.2f seconds.", _name.c_str(), duration.count());
return items;
return allItems;
}
std::tuple<bool, std::vector<TItem>> ReadIndexFile(const DirectoryStats &stats) const
@ -229,6 +295,7 @@ private:
header.Stats.FileDateModifiedChecksum == stats.FileDateModifiedChecksum &&
header.Stats.PathChecksum == stats.PathChecksum)
{
items.reserve(header.NumItems);
// Directory is the same, just read the saved items
for (uint32 i = 0; i < header.NumItems; i++)
{
@ -258,7 +325,7 @@ private:
log_verbose("FileIndex:Writing index: '%s'", _indexPath.c_str());
Path::CreateDirectory(Path::GetDirectory(_indexPath));
auto fs = FileStream(_indexPath, FILE_MODE_WRITE);
// Write header
FileIndexHeader header;
header.MagicNumber = _magicNumber;
@ -268,7 +335,7 @@ private:
header.Stats = stats;
header.NumItems = (uint32)items.size();
fs.WriteValue(header);
// Write items
for (const auto item : items)
{

176
src/openrct2/core/JobPool.hpp Normal file
View File

@ -0,0 +1,176 @@
#pragma region Copyright (c) 2014-2017 OpenRCT2 Developers
/*****************************************************************************
* OpenRCT2, an open source clone of Roller Coaster Tycoon 2.
*
* OpenRCT2 is the work of many authors, a full list can be found in contributors.md
* For more information, visit https://github.com/OpenRCT2/OpenRCT2
*
* OpenRCT2 is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* A full copy of the GNU General Public License can be found in licence.txt
*****************************************************************************/
#pragma endregion
#pragma once
#include <atomic>
#include <cassert>
#include <condition_variable>
#include <deque>
#include <functional>
#include <mutex>
#include <thread>
#include <vector>
class JobPool
{
private:
struct TaskData
{
const std::function<void()> WorkFn;
const std::function<void()> CompletionFn;
TaskData(std::function<void()> workFn, std::function<void()> completionFn)
: WorkFn(workFn),
CompletionFn(completionFn)
{
}
};
std::atomic_bool _shouldStop = { false };
std::atomic<size_t> _processing = { 0 };
std::vector<std::thread> _threads;
std::deque<TaskData> _pending;
std::deque<TaskData> _completed;
std::condition_variable _condPending;
std::condition_variable _condComplete;
std::mutex _mutex;
typedef std::unique_lock<std::mutex> unique_lock;
public:
JobPool()
{
for (size_t n = 0; n < std::thread::hardware_concurrency(); n++)
{
_threads.emplace_back(&JobPool::ProcessQueue, this);
}
}
~JobPool()
{
{
unique_lock lock(_mutex);
_shouldStop = true;
_condPending.notify_all();
}
for (auto&& th : _threads)
{
assert(th.joinable() != false);
th.join();
}
}
void AddTask(std::function<void()> workFn, std::function<void()> completionFn)
{
unique_lock lock(_mutex);
_pending.emplace_back(workFn, completionFn);
_condPending.notify_one();
}
void AddTask(std::function<void()> workFn)
{
return AddTask(workFn, nullptr);
}
void Join(std::function<void()> reportFn = nullptr)
{
unique_lock lock(_mutex);
while (true)
{
// Wait for the queue to become empty or having completed tasks.
_condComplete.wait(lock, [this]()
{
return (_pending.empty() && _processing == 0) ||
!_completed.empty();
});
// Dispatch all completion callbacks if there are any.
while (!_completed.empty())
{
auto taskData = _completed.front();
_completed.pop_front();
if (taskData.CompletionFn)
{
lock.unlock();
taskData.CompletionFn();
lock.lock();
}
}
if (reportFn)
{
lock.unlock();
reportFn();
lock.lock();
}
// If everything is empty and no more work has to be done we can stop waiting.
if (_completed.empty() &&
_pending.empty() &&
_processing == 0)
{
break;
}
}
}
size_t CountPending()
{
return _pending.size();
}
private:
void ProcessQueue()
{
unique_lock lock(_mutex);
do
{
// Wait for work or cancelation.
_condPending.wait(lock,
[this]()
{
return _shouldStop || !_pending.empty();
});
if (!_pending.empty())
{
_processing++;
auto taskData = _pending.front();
_pending.pop_front();
lock.unlock();
taskData.WorkFn();
lock.lock();
_completed.push_back(taskData);
_processing--;
_condComplete.notify_one();
}
}
while(!_shouldStop);
}
};

8
src/openrct2/core/String.cpp
View File

@ -41,6 +41,14 @@ namespace String
else return std::string(str);
}
std::string StdFormat_VA(const utf8 * format, va_list args)
{
auto buffer = Format_VA(format, args);
auto returnValue = ToStd(buffer);
Memory::Free(buffer);
return returnValue;
}
std::string StdFormat(const utf8 * format, ...)
{
va_list args;

1
src/openrct2/core/String.hpp
View File

@ -40,6 +40,7 @@ namespace String
constexpr const utf8 * Empty = "";
std::string ToStd(const utf8 * str);
std::string StdFormat_VA(const utf8 * format, va_list args);
std::string StdFormat(const utf8 * format, ...);
std::string ToUtf8(const std::wstring &s);
std::wstring ToUtf16(const std::string &s);

32
test/testpaint/CMakeLists.txt
View File

@ -54,15 +54,28 @@ endif ()
set(OPENRCT2_SRCPATH "${ROOT_DIR}/src/openrct2")
file(GLOB_RECURSE ORCT2_RIDE_SOURCES "${OPENRCT2_SRCPATH}/ride/*/*.cpp")
file(GLOB_RECURSE ORCT2_RIDE_DEP_SOURCES "${OPENRCT2_SRCPATH}/ride/RideData.cpp"
"${OPENRCT2_SRCPATH}/ride/TrackData.cpp"
"${OPENRCT2_SRCPATH}/ride/TrackDataOld.cpp"
"${OPENRCT2_SRCPATH}/ride/TrackPaint.cpp"
"${OPENRCT2_SRCPATH}/rct2/addresses.c"
"${OPENRCT2_SRCPATH}/Diagnostic.cpp"
"${OPENRCT2_SRCPATH}/rct2/hook.c"
"${OPENRCT2_SRCPATH}/paint/tile_element/TileElement.cpp"
"${OPENRCT2_SRCPATH}/paint/PaintHelpers.cpp")
file(GLOB_RECURSE ORCT2_RIDE_DEP_SOURCES
"${OPENRCT2_SRCPATH}/Diagnostic.cpp"
"${OPENRCT2_SRCPATH}/paint/PaintHelpers.cpp"
"${OPENRCT2_SRCPATH}/paint/tile_element/TileElement.cpp"
"${OPENRCT2_SRCPATH}/rct2/addresses.c"
"${OPENRCT2_SRCPATH}/rct2/hook.c"
"${OPENRCT2_SRCPATH}/ride/RideData.cpp"
"${OPENRCT2_SRCPATH}/ride/TrackData.cpp"
"${OPENRCT2_SRCPATH}/ride/TrackDataOld.cpp"
"${OPENRCT2_SRCPATH}/ride/TrackPaint.cpp"
"${ROOT_DIR}/src/openrct2/core/Console.cpp"
"${ROOT_DIR}/src/openrct2/core/Diagnostics.cpp"
"${ROOT_DIR}/src/openrct2/core/Guard.cpp"
"${ROOT_DIR}/src/openrct2/core/String.cpp"
"${ROOT_DIR}/src/openrct2/Diagnostic.cpp"
"${ROOT_DIR}/src/openrct2/localisation/ConversionTables.cpp"
"${ROOT_DIR}/src/openrct2/localisation/Convert.cpp"
"${ROOT_DIR}/src/openrct2/localisation/FormatCodes.cpp"
"${ROOT_DIR}/src/openrct2/localisation/UTF8.cpp"
"${ROOT_DIR}/src/openrct2/util/Util.cpp"
"${ROOT_DIR}/src/openrct2/Version.cpp"
)
file(GLOB_RECURSE ORCT2_TESTPAINT_SOURCES "${CMAKE_CURRENT_LIST_DIR}/*.c"
"${CMAKE_CURRENT_LIST_DIR}/*.cpp"
"${CMAKE_CURRENT_LIST_DIR}/*.h")
@ -73,6 +86,7 @@ set_source_files_properties(${CMAKE_CURRENT_LIST_DIR}/addresses.c PROPERTIES COM
add_executable(testpaint EXCLUDE_FROM_ALL ${ORCT2_RIDE_SOURCES} ${ORCT2_RIDE_DEP_SOURCES} ${ORCT2_TESTPAINT_SOURCES} ${RCT2_SECTIONS})
target_include_directories(testpaint PRIVATE "${ROOT_DIR}/src/")
target_link_libraries(testpaint z)
set_target_properties(testpaint PROPERTIES COMPILE_FLAGS "-DNO_VEHICLES -D__TESTPAINT__ -Wno-unused")
set_target_properties(testpaint PROPERTIES LINK_FLAGS ${RCT2_SEGMENT_LINKER_FLAGS})