From 2646ac3f5629f83536dd42c1f847f6fe84a5dbb5 Mon Sep 17 00:00:00 2001 From: Ted John Date: Sat, 26 Aug 2017 18:19:30 +0100 Subject: [PATCH 01/16] Create index file for scenario repository Speeds up load time for game and scenario select window. --- src/openrct2/PlatformEnvironment.cpp | 1 + src/openrct2/PlatformEnvironment.h | 1 + src/openrct2/scenario/ScenarioRepository.cpp | 103 ++++++++++++++++++- 3 files changed, 101 insertions(+), 4 deletions(-) diff --git a/src/openrct2/PlatformEnvironment.cpp b/src/openrct2/PlatformEnvironment.cpp index b79eb7c749..abb711a938 100644 --- a/src/openrct2/PlatformEnvironment.cpp +++ b/src/openrct2/PlatformEnvironment.cpp @@ -186,6 +186,7 @@ const char * PlatformEnvironment::FileNames[] = "hotkeys.dat", // CONFIG_KEYBOARD "objects.idx", // CACHE_OBJECTS "tracks.idx", // CACHE_TRACKS + "scenarios.idx", // CACHE_SCENARIOS "RCTdeluxe_install" PATH_SEPARATOR "Data" PATH_SEPARATOR "mp.dat", // MP_DAT "groups.json", // NETWORK_GROUPS "servers.cfg", // NETWORK_SERVERS diff --git a/src/openrct2/PlatformEnvironment.h b/src/openrct2/PlatformEnvironment.h index 1996d5efa1..d0eeeae1fb 100644 --- a/src/openrct2/PlatformEnvironment.h +++ b/src/openrct2/PlatformEnvironment.h @@ -55,6 +55,7 @@ namespace OpenRCT2 CONFIG_KEYBOARD, // Keyboard shortcuts. (hotkeys.cfg) CACHE_OBJECTS, // Object repository cache (objects.idx). CACHE_TRACKS, // Track repository cache (tracks.idx). + CACHE_SCENARIOS, // Scenario repository cache (scenarios.idx). MP_DAT, // Mega Park data, Steam RCT1 only (\RCTdeluxe_install\Data\mp.dat) NETWORK_GROUPS, // Server groups with permissions (groups.json). NETWORK_SERVERS, // Saved servers (servers.cfg). diff --git a/src/openrct2/scenario/ScenarioRepository.cpp b/src/openrct2/scenario/ScenarioRepository.cpp index f25ba11c4f..9ce8368134 100644 --- a/src/openrct2/scenario/ScenarioRepository.cpp +++ b/src/openrct2/scenario/ScenarioRepository.cpp @@ -41,6 +41,20 @@ extern "C" using namespace OpenRCT2; +constexpr uint32 SCENARIO_REPOSITORY_MAGIC_NUMBER = 0x58444953; +constexpr uint16 SCENARIO_REPOSITORY_VERSION = 1; + +struct ScenarioRepositoryHeader +{ + uint32 MagicNumber; + uint16 Version; + uint32 TotalFiles; + uint64 TotalFileSize; + uint32 FileDateModifiedChecksum; + uint32 PathChecksum; + uint32 NumItems; +}; + static sint32 ScenarioCategoryCompare(sint32 categoryA, sint32 categoryB) { if (categoryA == categoryB) return 0; @@ -125,11 +139,13 @@ static void scenario_highscore_free(scenario_highscore_entry * highscore) class ScenarioRepository final : public IScenarioRepository { private: + static constexpr const utf8 * SC_FILE_PATTERN = "*.sc4;*.sc6"; static constexpr uint32 HighscoreFileVersion = 1; IPlatformEnvironment * _env; std::vector _scenarios; std::vector _highscores; + QueryDirectoryResult _directoryQueryResult = { 0 }; public: ScenarioRepository(IPlatformEnvironment * env) @@ -152,13 +168,23 @@ public: std::string openrct2dir = _env->GetDirectoryPath(DIRBASE::USER, DIRID::SCENARIO); std::string mpdatdir = _env->GetFilePath(PATHID::MP_DAT); - Scan(rct1dir); - Scan(rct2dir); - Scan(openrct2dir); + _directoryQueryResult = { 0 }; + Query(rct1dir); + Query(rct2dir); + Query(openrct2dir); + + if (!Load()) + { + Scan(rct1dir); + Scan(rct2dir); + Scan(openrct2dir); + Save(); + } ConvertMegaPark(mpdatdir, openrct2dir); Sort(); + LoadScores(); LoadLegacyScores(); AttachHighscores(); @@ -260,11 +286,17 @@ private: return (scenario_index_entry *)repo->GetByPath(path); } + void Query(const std::string &directory) + { + std::string pattern = Path::Combine(directory, SC_FILE_PATTERN); + Path::QueryDirectory(&_directoryQueryResult, pattern); + } + void Scan(const std::string &directory) { utf8 pattern[MAX_PATH]; String::Set(pattern, sizeof(pattern), directory.c_str()); - Path::Append(pattern, sizeof(pattern), "*.sc4;*.sc6"); + Path::Append(pattern, sizeof(pattern), SC_FILE_PATTERN); IFileScanner * scanner = Path::ScanDirectory(pattern, true); while (scanner->Next()) @@ -471,6 +503,69 @@ private: } } + bool Load() + { + std::string path = _env->GetFilePath(PATHID::CACHE_SCENARIOS); + bool result = false; + try + { + auto fs = FileStream(path, FILE_MODE_OPEN); + + // Read header, check if we need to re-scan + auto header = fs.ReadValue(); + if (header.MagicNumber == SCENARIO_REPOSITORY_MAGIC_NUMBER && + header.Version == SCENARIO_REPOSITORY_VERSION && + header.TotalFiles == _directoryQueryResult.TotalFiles && + header.TotalFileSize == _directoryQueryResult.TotalFileSize && + header.FileDateModifiedChecksum == _directoryQueryResult.FileDateModifiedChecksum && + header.PathChecksum == _directoryQueryResult.PathChecksum) + { + // Directory is the same, just read the saved items + for (uint32 i = 0; i < header.NumItems; i++) + { + auto scenario = fs.ReadValue(); + _scenarios.push_back(scenario); + } + result = true; + } + } + catch (const Exception &) + { + Console::Error::WriteLine("Unable to load scenario repository index."); + } + return result; + } + + void Save() const + { + std::string path = _env->GetFilePath(PATHID::CACHE_SCENARIOS); + try + { + auto fs = FileStream(path, FILE_MODE_WRITE); + + // Write header + ScenarioRepositoryHeader header = { 0 }; + header.MagicNumber = SCENARIO_REPOSITORY_MAGIC_NUMBER; + header.Version = SCENARIO_REPOSITORY_VERSION; + header.TotalFiles = _directoryQueryResult.TotalFiles; + header.TotalFileSize = _directoryQueryResult.TotalFileSize; + header.FileDateModifiedChecksum = _directoryQueryResult.FileDateModifiedChecksum; + header.PathChecksum = _directoryQueryResult.PathChecksum; + header.NumItems = (uint32)_scenarios.size(); + fs.WriteValue(header); + + // Write items + for (const auto scenario : _scenarios) + { + fs.WriteValue(scenario); + } + } + catch (const Exception &) + { + Console::Error::WriteLine("Unable to write scenario repository index."); + } + } + void LoadScores() { std::string path = _env->GetFilePath(PATHID::SCORES); From aad2cb60e2532be028a53b5500555a4bac4decd8 Mon Sep 17 00:00:00 2001 From: Ted John Date: Sat, 26 Aug 2017 18:21:26 +0100 Subject: [PATCH 02/16] Update changelog [ci skip] --- distribution/changelog.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/distribution/changelog.txt b/distribution/changelog.txt index fadb065552..7a447ba347 100644 --- a/distribution/changelog.txt +++ b/distribution/changelog.txt @@ -15,6 +15,7 @@ - Fix: [#6198] You cannot cancel RCT1 directory selection - Fix: Infinite loop when removing scenery elements with >127 base height. - Improved: [#6186] Transparent menu items now draw properly in OpenGL mode. +- Improved: [#6218] Speed up game start up time by saving scenario index to file. - Improved: Load/save window now refreshes list if native file dialog is closed/cancelled 0.1.1 (2017-08-09) From 4f3669f2791a28ab3f70efd2f80e47c146324e6c Mon Sep 17 00:00:00 2001 From: Ted John Date: Tue, 29 Aug 2017 19:18:13 +0100 Subject: [PATCH 03/16] Start work on new FileIndex base class [ci skip] --- src/openrct2/core/FileIndex.hpp | 209 ++++++++++++++++++++++++++++++++ 1 file changed, 209 insertions(+) create mode 100644 src/openrct2/core/FileIndex.hpp diff --git a/src/openrct2/core/FileIndex.hpp b/src/openrct2/core/FileIndex.hpp new file mode 100644 index 0000000000..d206c32394 --- /dev/null +++ b/src/openrct2/core/FileIndex.hpp @@ -0,0 +1,209 @@ +#pragma region Copyright (c) 2017 OpenRCT2 Developers +/***************************************************************************** + * OpenRCT2, an open source clone of Roller Coaster Tycoon 2. + * + * OpenRCT2 is the work of many authors, a full list can be found in contributors.md + * For more information, visit https://github.com/OpenRCT2/OpenRCT2 + * + * OpenRCT2 is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * A full copy of the GNU General Public License can be found in licence.txt + *****************************************************************************/ +#pragma endregion + +#pragma once + +#include +#include "../common.h" + +template +class FileIndex +{ +private: + struct DirectoryStats + { + uint32 TotalFiles = 0; + uint64 TotalFileSize = 0; + uint32 FileDateModifiedChecksum = 0; + uint32 PathChecksum = 0; + } + + struct ScanResult + { + DirectoryStats const Stats; + std::vector const Files; + + ScanResult(DirectoryStats stats, std::vector files) + : Stats(stats), + Files(files) + { + } + }; + + struct FileIndexHeader + { + uint32 MagicNumber = 0; + uint8 VersionA = 0; + uint8 VersionB = 0; + uint16 LanguageId = 0; + DirectoryStats DirectoryStats; + uint32 NumItems = 0; + }; + + constexpr uint8 FILE_INDEX_VERSION = 4; + + uint32 _magicNumber; + uint8 _version; + std::string _indexPath; + std::string _pattern; + std::vector _paths; + +public: + FileIndex(uint32 magicNumber, + uint8 version, + std::string indexPath, + std::string pattern, + std::vector paths) : + _magicNumber(magicNumber), + _version(version), + _indexPath(indexPath), + _pattern(pattern), + _paths(paths) + { + } + + /** + * Queries and directories and loads the index header. If the index is up to date, + * the items are loaded from the index and returned, otherwise the index is rebuilt. + */ + std::vector LoadOrBuild() + { + std::vector items; + auto scanResult = Scan(); + auto readIndexResult = ReadIndexFile(scanResult.Stats); + if (std::get<0>(readIndexResult)) + { + // Index was loaded + items = std::get<1>(readIndexResult); + } + else + { + // Index was not loaded + for (auto filePath : scanResult.Files) + { + auto item = Create(filePath); + items.push_back(item); + } + WriteIndexFile(items); + } + return items; + } + + /** + * Loads the given file and creates the item representing the data to store in the index. + */ + virtual TItem Create(const std::string &path) abstract; + + /** + * Serialises an index item to the given stream. + */ + virtual void Serialise(IStream * stream, const TItem item); + + /** + * Deserialises an index item from the given stream. + */ + virtual TItem Deserialise(IStream * stream) abstract; + +private: + ScanResult Scan() + { + ScanResult scanResult; + for (const auto directory : _paths) + { + auto pattern = Path::Combine(directory, _pattern); + auto scanner = Path::ScanDirectory(pattern, true); + while (scanner->Next()) + { + auto fileInfo = scanner->GetFileInfo(); + auto path = std::string(scanner->GetPath()); + + scanResult.Files.push(path); + + scanResult.TotalFiles++; + scanResult.TotalFileSize += fileInfo->Size; + scanResult.FileDateModifiedChecksum ^= + (uint32)(fileInfo->LastModified >> 32) ^ + (uint32)(fileInfo->LastModified & 0xFFFFFFFF); + scanResult.FileDateModifiedChecksum = ror32(result->FileDateModifiedChecksum, 5); + scanResult.PathChecksum += GetPathChecksum(path); + } + delete scanner; + } + } + + std::tuple> ReadIndexFile(const DirectoryStats &stats) + { + bool loadedItems = false; + std::vector items; + try + { + auto fs = FileStream(path, FILE_MODE_OPEN); + + // Read header, check if we need to re-scan + auto header = fs.ReadValue(); + if (header.MagicNumber == _magicNumber && + header.VersionA == FILE_INDEX_VERSION && + header.VersionB == _version && + header.TotalFiles == scanResult.TotalFiles && + header.TotalFileSize == scanResult.TotalFileSize && + header.FileDateModifiedChecksum == scanResult.FileDateModifiedChecksum && + header.PathChecksum == scanResult.PathChecksum) + { + // Directory is the same, just read the saved items + for (uint32 i = 0; i < header.NumItems; i++) + { + auto item = Deserialise(fs); + items.push_back(item); + } + loadedItems = true; + } + } + catch (const Exception &) + { + Console::Error::WriteLine("Unable to load index."); + } + return std::make_tuple(loadedItems, items); + } + + void WriteIndexFile(const DirectoryStats &stats, const std::vector &items) + { + try + { + auto fs = FileStream(_indexPath, FILE_MODE_WRITE); + + // Write header + FileIndexHeader header = { 0 }; + header.MagicNumber = _magicNumber; + header.Version = SCENARIO_REPOSITORY_VERSION; + header.LanguageId = gCurrentLanguage; + header.DirectoryStats = stats; + header.NumItems = items.size(); + fs.WriteValue(header); + + // Write items + for (const auto item : items) + { + Serialise(fs, item); + } + return true; + } + catch (const Exception &) + { + Console::Error::WriteLine("Unable to save index."); + return false; + } + } +}; From e417d2f8b010b65eabe890fb2bd1d7e5125619e6 Mon Sep 17 00:00:00 2001 From: Ted John Date: Tue, 29 Aug 2017 23:25:52 +0100 Subject: [PATCH 04/16] Fix all errors --- src/openrct2/core/File.cpp | 33 ++ src/openrct2/core/File.h | 1 + src/openrct2/core/FileIndex.hpp | 95 +++-- src/openrct2/scenario/ScenarioRepository.cpp | 398 ++++++++----------- 4 files changed, 267 insertions(+), 260 deletions(-) diff --git a/src/openrct2/core/File.cpp b/src/openrct2/core/File.cpp index b33650b124..ed43400c85 100644 --- a/src/openrct2/core/File.cpp +++ b/src/openrct2/core/File.cpp @@ -14,6 +14,13 @@ *****************************************************************************/ #pragma endregion +#ifdef _WIN32 + #define WIN32_LEAN_AND_MEAN + #include +#else + #include +#endif + #include "Console.hpp" #include "File.h" #include "FileStream.hpp" @@ -103,6 +110,32 @@ namespace File Memory::Free(data); return lines; } + + uint64 GetLastModified(const std::string &path) + { + uint64 lastModified = 0; +#ifdef _WIN32 + auto pathW = utf8_to_widechar(path.c_str()); + auto hFile = CreateFileW(pathW, GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr); + if (hFile != INVALID_HANDLE_VALUE) + { + FILETIME ftCreate, ftAccess, ftWrite; + if (GetFileTime(hFile, &ftCreate, &ftAccess, &ftWrite)) + { + lastModified = ((uint64)ftWrite.dwHighDateTime << 32ULL) | (uint64)ftWrite.dwLowDateTime; + } + CloseHandle(hFile); + } + free(pathW); +#else + struct stat statInfo; + if (stat(path.c_str(), &statInfo) == 0) + { + lastModified = statInfo.st_mtime; + } +#endif + return lastModified; + } } extern "C" diff --git a/src/openrct2/core/File.h b/src/openrct2/core/File.h index d15481b3b3..2084be8608 100644 --- a/src/openrct2/core/File.h +++ b/src/openrct2/core/File.h @@ -29,4 +29,5 @@ namespace File void * ReadAllBytes(const std::string &path, size_t * length); void WriteAllBytes(const std::string &path, const void * buffer, size_t length); std::vector ReadAllLines(const std::string &path); + uint64 GetLastModified(const std::string &path); } diff --git a/src/openrct2/core/FileIndex.hpp b/src/openrct2/core/FileIndex.hpp index d206c32394..05766786ce 100644 --- a/src/openrct2/core/FileIndex.hpp +++ b/src/openrct2/core/FileIndex.hpp @@ -17,7 +17,13 @@ #pragma once #include +#include +#include #include "../common.h" +#include "File.h" +#include "FileScanner.h" +#include "FileStream.hpp" +#include "Path.hpp" template class FileIndex @@ -29,7 +35,7 @@ private: uint64 TotalFileSize = 0; uint32 FileDateModifiedChecksum = 0; uint32 PathChecksum = 0; - } + }; struct ScanResult { @@ -49,17 +55,17 @@ private: uint8 VersionA = 0; uint8 VersionB = 0; uint16 LanguageId = 0; - DirectoryStats DirectoryStats; + DirectoryStats Stats; uint32 NumItems = 0; }; - constexpr uint8 FILE_INDEX_VERSION = 4; + static constexpr uint8 FILE_INDEX_VERSION = 4; - uint32 _magicNumber; - uint8 _version; - std::string _indexPath; - std::string _pattern; - std::vector _paths; + uint32 const _magicNumber; + uint8 const _version; + std::string const _indexPath; + std::string const _pattern; + std::vector const _paths; public: FileIndex(uint32 magicNumber, @@ -75,11 +81,13 @@ public: { } + virtual ~FileIndex() = default; + /** * Queries and directories and loads the index header. If the index is up to date, * the items are loaded from the index and returned, otherwise the index is rebuilt. */ - std::vector LoadOrBuild() + std::vector LoadOrBuild() const { std::vector items; auto scanResult = Scan(); @@ -97,30 +105,32 @@ public: auto item = Create(filePath); items.push_back(item); } - WriteIndexFile(items); + WriteIndexFile(scanResult.Stats, items); } return items; } +protected: /** * Loads the given file and creates the item representing the data to store in the index. */ - virtual TItem Create(const std::string &path) abstract; + virtual TItem Create(const std::string &path) const abstract; /** * Serialises an index item to the given stream. */ - virtual void Serialise(IStream * stream, const TItem item); + virtual void Serialise(IStream * stream, const TItem &item) const abstract; /** * Deserialises an index item from the given stream. */ - virtual TItem Deserialise(IStream * stream) abstract; + virtual TItem Deserialise(IStream * stream) const abstract; private: - ScanResult Scan() + ScanResult Scan() const { - ScanResult scanResult; + DirectoryStats stats; + std::vector files; for (const auto directory : _paths) { auto pattern = Path::Combine(directory, _pattern); @@ -130,42 +140,43 @@ private: auto fileInfo = scanner->GetFileInfo(); auto path = std::string(scanner->GetPath()); - scanResult.Files.push(path); + files.push_back(path); - scanResult.TotalFiles++; - scanResult.TotalFileSize += fileInfo->Size; - scanResult.FileDateModifiedChecksum ^= + stats.TotalFiles++; + stats.TotalFileSize += fileInfo->Size; + stats.FileDateModifiedChecksum ^= (uint32)(fileInfo->LastModified >> 32) ^ (uint32)(fileInfo->LastModified & 0xFFFFFFFF); - scanResult.FileDateModifiedChecksum = ror32(result->FileDateModifiedChecksum, 5); - scanResult.PathChecksum += GetPathChecksum(path); + stats.FileDateModifiedChecksum = ror32(stats.FileDateModifiedChecksum, 5); + stats.PathChecksum += GetPathChecksum(path); } delete scanner; } + return ScanResult(stats, files); } - std::tuple> ReadIndexFile(const DirectoryStats &stats) + std::tuple> ReadIndexFile(const DirectoryStats &stats) const { bool loadedItems = false; std::vector items; try { - auto fs = FileStream(path, FILE_MODE_OPEN); + auto fs = FileStream(_indexPath, FILE_MODE_OPEN); // Read header, check if we need to re-scan auto header = fs.ReadValue(); if (header.MagicNumber == _magicNumber && header.VersionA == FILE_INDEX_VERSION && header.VersionB == _version && - header.TotalFiles == scanResult.TotalFiles && - header.TotalFileSize == scanResult.TotalFileSize && - header.FileDateModifiedChecksum == scanResult.FileDateModifiedChecksum && - header.PathChecksum == scanResult.PathChecksum) + header.Stats.TotalFiles == stats.TotalFiles && + header.Stats.TotalFileSize == stats.TotalFileSize && + header.Stats.FileDateModifiedChecksum == stats.FileDateModifiedChecksum && + header.Stats.PathChecksum == stats.PathChecksum) { // Directory is the same, just read the saved items for (uint32 i = 0; i < header.NumItems; i++) { - auto item = Deserialise(fs); + auto item = Deserialise(&fs); items.push_back(item); } loadedItems = true; @@ -178,7 +189,7 @@ private: return std::make_tuple(loadedItems, items); } - void WriteIndexFile(const DirectoryStats &stats, const std::vector &items) + void WriteIndexFile(const DirectoryStats &stats, const std::vector &items) const { try { @@ -187,23 +198,37 @@ private: // Write header FileIndexHeader header = { 0 }; header.MagicNumber = _magicNumber; - header.Version = SCENARIO_REPOSITORY_VERSION; + header.VersionA = FILE_INDEX_VERSION; + header.VersionB = _version; header.LanguageId = gCurrentLanguage; - header.DirectoryStats = stats; - header.NumItems = items.size(); + header.Stats = stats; + header.NumItems = (uint32)items.size(); fs.WriteValue(header); // Write items for (const auto item : items) { - Serialise(fs, item); + Serialise(&fs, item); } - return true; } catch (const Exception &) { Console::Error::WriteLine("Unable to save index."); - return false; } } + + static uint32 GetPathChecksum(const std::string &path) + { + uint32 hash = 0xD8430DED; + for (const utf8 * ch = path.c_str(); *ch != '\0'; ch++) + { + hash += (*ch); + hash += (hash << 10); + hash ^= (hash >> 6); + } + hash += (hash << 3); + hash ^= (hash >> 11); + hash += (hash << 15); + return hash; + } }; diff --git a/src/openrct2/scenario/ScenarioRepository.cpp b/src/openrct2/scenario/ScenarioRepository.cpp index 9ce8368134..af3b5e5c67 100644 --- a/src/openrct2/scenario/ScenarioRepository.cpp +++ b/src/openrct2/scenario/ScenarioRepository.cpp @@ -19,6 +19,7 @@ #include #include "../core/Console.hpp" #include "../core/File.h" +#include "../core/FileIndex.hpp" #include "../core/FileScanner.h" #include "../core/FileStream.hpp" #include "../core/Math.hpp" @@ -136,6 +137,165 @@ static void scenario_highscore_free(scenario_highscore_entry * highscore) SafeDelete(highscore); } +class ScenarioFileIndex final : public FileIndex +{ +private: + static constexpr uint32 MAGIC_NUMBER = 0x58444953; + static constexpr uint16 VERSION = 1; + static constexpr auto PATTERN = "*.sc4;*.sc6"; + +public: + ScenarioFileIndex(IPlatformEnvironment * env) : + FileIndex(MAGIC_NUMBER, + VERSION, + env->GetFilePath(PATHID::CACHE_SCENARIOS), + std::string(PATTERN), + std::vector({ + env->GetDirectoryPath(DIRBASE::RCT1, DIRID::SCENARIO), + env->GetDirectoryPath(DIRBASE::RCT2, DIRID::SCENARIO), + env->GetDirectoryPath(DIRBASE::USER, DIRID::SCENARIO) })) + { + } + +protected: + scenario_index_entry Create(const std::string &path) const override + { + scenario_index_entry entry; + auto timestamp = File::GetLastModified(path); + if (!GetScenarioInfo(path, timestamp, &entry)) + { + // TODO + } + return entry; + } + + void Serialise(IStream * stream, const scenario_index_entry &item) const override + { + // HACK: Zero highscore pointer + auto copy = item; + copy.highscore = nullptr; + stream->WriteValue(copy); + } + + scenario_index_entry Deserialise(IStream * stream) const override + { + auto result = stream->ReadValue(); + // HACK: Zero highscore pointer + result.highscore = nullptr; + return result; + } + +private: + /** + * Reads basic information from a scenario file. + */ + static bool GetScenarioInfo(const std::string &path, uint64 timestamp, scenario_index_entry * entry) + { + log_verbose("GetScenarioInfo(%s, %d, ...)", path.c_str(), timestamp); + try + { + std::string extension = Path::GetExtension(path); + if (String::Equals(extension, ".sc4", true)) + { + // RCT1 scenario + bool result = false; + try + { + auto s4Importer = std::unique_ptr(ParkImporter::CreateS4()); + s4Importer->LoadScenario(path.c_str(), true); + if (s4Importer->GetDetails(entry)) + { + String::Set(entry->path, sizeof(entry->path), path.c_str()); + entry->timestamp = timestamp; + result = true; + } + } + catch (Exception) + { + } + return result; + } + else + { + // RCT2 scenario + auto fs = FileStream(path, FILE_MODE_OPEN); + auto chunkReader = SawyerChunkReader(&fs); + + rct_s6_header header = chunkReader.ReadChunkAs(); + if (header.type == S6_TYPE_SCENARIO) + { + rct_s6_info info = chunkReader.ReadChunkAs(); + *entry = CreateNewScenarioEntry(path, timestamp, &info); + return true; + } + else + { + log_verbose("%s is not a scenario", path.c_str()); + } + } + } + catch (Exception) + { + Console::Error::WriteLine("Unable to read scenario: '%s'", path.c_str()); + } + return false; + } + + static scenario_index_entry CreateNewScenarioEntry(const std::string &path, uint64 timestamp, rct_s6_info * s6Info) + { + scenario_index_entry entry = { 0 }; + + // Set new entry + String::Set(entry.path, sizeof(entry.path), path.c_str()); + entry.timestamp = timestamp; + entry.category = s6Info->category; + entry.objective_type = s6Info->objective_type; + entry.objective_arg_1 = s6Info->objective_arg_1; + entry.objective_arg_2 = s6Info->objective_arg_2; + entry.objective_arg_3 = s6Info->objective_arg_3; + entry.highscore = nullptr; + if (String::IsNullOrEmpty(s6Info->name)) + { + // If the scenario doesn't have a name, set it to the filename + String::Set(entry.name, sizeof(entry.name), Path::GetFileNameWithoutExtension(entry.path)); + } + else + { + String::Set(entry.name, sizeof(entry.name), s6Info->name); + // Normalise the name to make the scenario as recognisable as possible. + ScenarioSources::NormaliseName(entry.name, sizeof(entry.name), entry.name); + } + + String::Set(entry.details, sizeof(entry.details), s6Info->details); + + // Look up and store information regarding the origins of this scenario. + source_desc desc; + if (ScenarioSources::TryGetByName(entry.name, &desc)) + { + entry.sc_id = desc.id; + entry.source_index = desc.index; + entry.source_game = desc.source; + entry.category = desc.category; + } + else + { + entry.sc_id = SC_UNIDENTIFIED; + entry.source_index = -1; + if (entry.category == SCENARIO_CATEGORY_REAL) + { + entry.source_game = SCENARIO_SOURCE_REAL; + } + else + { + entry.source_game = SCENARIO_SOURCE_OTHER; + } + } + + scenario_translate(&entry, &s6Info->entry); + return entry; + } +}; + class ScenarioRepository final : public IScenarioRepository { private: @@ -143,12 +303,13 @@ private: static constexpr uint32 HighscoreFileVersion = 1; IPlatformEnvironment * _env; + ScenarioFileIndex const _fileIndex; std::vector _scenarios; std::vector _highscores; - QueryDirectoryResult _directoryQueryResult = { 0 }; public: ScenarioRepository(IPlatformEnvironment * env) + : _fileIndex(env) { _env = env; } @@ -162,29 +323,16 @@ public: { _scenarios.clear(); - // Scan RCT2 directory - std::string rct1dir = _env->GetDirectoryPath(DIRBASE::RCT1, DIRID::SCENARIO); - std::string rct2dir = _env->GetDirectoryPath(DIRBASE::RCT2, DIRID::SCENARIO); - std::string openrct2dir = _env->GetDirectoryPath(DIRBASE::USER, DIRID::SCENARIO); - std::string mpdatdir = _env->GetFilePath(PATHID::MP_DAT); - - _directoryQueryResult = { 0 }; - Query(rct1dir); - Query(rct2dir); - Query(openrct2dir); - - if (!Load()) + auto scenarios = _fileIndex.LoadOrBuild(); + for (auto scenario : scenarios) { - Scan(rct1dir); - Scan(rct2dir); - Scan(openrct2dir); - Save(); + AddScenario(scenario); } - - ConvertMegaPark(mpdatdir, openrct2dir); + + // std::string mpdatdir = _env->GetFilePath(PATHID::MP_DAT); + // ConvertMegaPark(mpdatdir, openrct2dir); Sort(); - LoadScores(); LoadLegacyScores(); AttachHighscores(); @@ -286,28 +434,6 @@ private: return (scenario_index_entry *)repo->GetByPath(path); } - void Query(const std::string &directory) - { - std::string pattern = Path::Combine(directory, SC_FILE_PATTERN); - Path::QueryDirectory(&_directoryQueryResult, pattern); - } - - void Scan(const std::string &directory) - { - utf8 pattern[MAX_PATH]; - String::Set(pattern, sizeof(pattern), directory.c_str()); - Path::Append(pattern, sizeof(pattern), SC_FILE_PATTERN); - - IFileScanner * scanner = Path::ScanDirectory(pattern, true); - while (scanner->Next()) - { - auto path = scanner->GetPath(); - auto fileInfo = scanner->GetFileInfo(); - AddScenario(path, fileInfo->LastModified); - } - delete scanner; - } - void ConvertMegaPark(std::string &mpdatDir, std::string &scenarioDir) { //Convert mp.dat from RCT1 Data directory into SC21.SC4 (Mega Park) @@ -340,20 +466,14 @@ private: } } - void AddScenario(const std::string &path, uint64 timestamp) + void AddScenario(const scenario_index_entry &entry) { - scenario_index_entry entry; - if (!GetScenarioInfo(path, timestamp, &entry)) - { - return; - } - - const std::string filename = Path::GetFileName(path); - scenario_index_entry * existingEntry = GetByFilename(filename.c_str()); + auto filename = Path::GetFileName(entry.path); + auto existingEntry = GetByFilename(filename); if (existingEntry != nullptr) { std::string conflictPath; - if (existingEntry->timestamp > timestamp) + if (existingEntry->timestamp > entry.timestamp) { // Existing entry is more recent conflictPath = String::ToStd(existingEntry->path); @@ -364,7 +484,7 @@ private: else { // This entry is more recent - conflictPath = path; + conflictPath = entry.path; } Console::WriteLine("Scenario conflict: '%s' ignored because it is newer.", conflictPath.c_str()); } @@ -374,115 +494,6 @@ private: } } - /** - * Reads basic information from a scenario file. - */ - bool GetScenarioInfo(const std::string &path, uint64 timestamp, scenario_index_entry * entry) - { - log_verbose("GetScenarioInfo(%s, %d, ...)", path.c_str(), timestamp); - try - { - std::string extension = Path::GetExtension(path); - if (String::Equals(extension, ".sc4", true)) - { - // RCT1 scenario - bool result = false; - try - { - auto s4Importer = std::unique_ptr(ParkImporter::CreateS4()); - s4Importer->LoadScenario(path.c_str(), true); - if (s4Importer->GetDetails(entry)) - { - String::Set(entry->path, sizeof(entry->path), path.c_str()); - entry->timestamp = timestamp; - result = true; - } - } - catch (Exception) - { - } - return result; - } - else - { - // RCT2 scenario - auto fs = FileStream(path, FILE_MODE_OPEN); - auto chunkReader = SawyerChunkReader(&fs); - - rct_s6_header header = chunkReader.ReadChunkAs(); - if (header.type == S6_TYPE_SCENARIO) - { - rct_s6_info info = chunkReader.ReadChunkAs(); - *entry = CreateNewScenarioEntry(path, timestamp, &info); - return true; - } - else - { - log_verbose("%s is not a scenario", path.c_str()); - } - } - } - catch (Exception) - { - Console::Error::WriteLine("Unable to read scenario: '%s'", path.c_str()); - } - return false; - } - - scenario_index_entry CreateNewScenarioEntry(const std::string &path, uint64 timestamp, rct_s6_info * s6Info) - { - scenario_index_entry entry = { 0 }; - - // Set new entry - String::Set(entry.path, sizeof(entry.path), path.c_str()); - entry.timestamp = timestamp; - entry.category = s6Info->category; - entry.objective_type = s6Info->objective_type; - entry.objective_arg_1 = s6Info->objective_arg_1; - entry.objective_arg_2 = s6Info->objective_arg_2; - entry.objective_arg_3 = s6Info->objective_arg_3; - entry.highscore = nullptr; - if (String::IsNullOrEmpty(s6Info->name)) - { - // If the scenario doesn't have a name, set it to the filename - String::Set(entry.name, sizeof(entry.name), Path::GetFileNameWithoutExtension(entry.path)); - } - else - { - String::Set(entry.name, sizeof(entry.name), s6Info->name); - // Normalise the name to make the scenario as recognisable as possible. - ScenarioSources::NormaliseName(entry.name, sizeof(entry.name), entry.name); - } - - String::Set(entry.details, sizeof(entry.details), s6Info->details); - - // Look up and store information regarding the origins of this scenario. - source_desc desc; - if (ScenarioSources::TryGetByName(entry.name, &desc)) - { - entry.sc_id = desc.id; - entry.source_index = desc.index; - entry.source_game = desc.source; - entry.category = desc.category; - } - else - { - entry.sc_id = SC_UNIDENTIFIED; - entry.source_index = -1; - if (entry.category == SCENARIO_CATEGORY_REAL) - { - entry.source_game = SCENARIO_SOURCE_REAL; - } - else - { - entry.source_game = SCENARIO_SOURCE_OTHER; - } - } - - scenario_translate(&entry, &s6Info->entry); - return entry; - } - void Sort() { if (gConfigGeneral.scenario_select_mode == SCENARIO_SELECT_MODE_ORIGIN) @@ -503,69 +514,6 @@ private: } } - bool Load() - { - std::string path = _env->GetFilePath(PATHID::CACHE_SCENARIOS); - bool result = false; - try - { - auto fs = FileStream(path, FILE_MODE_OPEN); - - // Read header, check if we need to re-scan - auto header = fs.ReadValue(); - if (header.MagicNumber == SCENARIO_REPOSITORY_MAGIC_NUMBER && - header.Version == SCENARIO_REPOSITORY_VERSION && - header.TotalFiles == _directoryQueryResult.TotalFiles && - header.TotalFileSize == _directoryQueryResult.TotalFileSize && - header.FileDateModifiedChecksum == _directoryQueryResult.FileDateModifiedChecksum && - header.PathChecksum == _directoryQueryResult.PathChecksum) - { - // Directory is the same, just read the saved items - for (uint32 i = 0; i < header.NumItems; i++) - { - auto scenario = fs.ReadValue(); - _scenarios.push_back(scenario); - } - result = true; - } - } - catch (const Exception &) - { - Console::Error::WriteLine("Unable to load scenario repository index."); - } - return result; - } - - void Save() const - { - std::string path = _env->GetFilePath(PATHID::CACHE_SCENARIOS); - try - { - auto fs = FileStream(path, FILE_MODE_WRITE); - - // Write header - ScenarioRepositoryHeader header = { 0 }; - header.MagicNumber = SCENARIO_REPOSITORY_MAGIC_NUMBER; - header.Version = SCENARIO_REPOSITORY_VERSION; - header.TotalFiles = _directoryQueryResult.TotalFiles; - header.TotalFileSize = _directoryQueryResult.TotalFileSize; - header.FileDateModifiedChecksum = _directoryQueryResult.FileDateModifiedChecksum; - header.PathChecksum = _directoryQueryResult.PathChecksum; - header.NumItems = (uint32)_scenarios.size(); - fs.WriteValue(header); - - // Write items - for (const auto scenario : _scenarios) - { - fs.WriteValue(scenario); - } - } - catch (const Exception &) - { - Console::Error::WriteLine("Unable to write scenario repository index."); - } - } - void LoadScores() { std::string path = _env->GetFilePath(PATHID::SCORES); From c5760a22bd47fa57c4504d88d1f0530c5aeb3a70 Mon Sep 17 00:00:00 2001 From: Ted John Date: Wed, 30 Aug 2017 18:57:11 +0100 Subject: [PATCH 05/16] Add more logging and comments --- src/openrct2/core/FileIndex.hpp | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src/openrct2/core/FileIndex.hpp b/src/openrct2/core/FileIndex.hpp index 05766786ce..0e4f996847 100644 --- a/src/openrct2/core/FileIndex.hpp +++ b/src/openrct2/core/FileIndex.hpp @@ -59,15 +59,28 @@ private: uint32 NumItems = 0; }; + // Index file format version which when incremented forces a rebuild static constexpr uint8 FILE_INDEX_VERSION = 4; + // Magic number for the index (to distinguish) uint32 const _magicNumber; + // Version for the specialised index uint8 const _version; + // Path to save the index at std::string const _indexPath; + // Pattern std::string const _pattern; std::vector const _paths; public: + /** + * Creates a new FileIndex. + * @param magicNumber Magic number for the index (to distinguish between different index files). + * @param version Version of the specialised index, increment this to force a rebuild. + * @param indexPath Full path to read and write the index file to. + * @param pattern The search pattern for indexing files. + * @param paths A list of search directories. + */ FileIndex(uint32 magicNumber, uint8 version, std::string indexPath, @@ -102,6 +115,7 @@ public: // Index was not loaded for (auto filePath : scanResult.Files) { + log_verbose("FileIndex:Indexing '%s'", filePath.c_str()); auto item = Create(filePath); items.push_back(item); } @@ -133,6 +147,8 @@ private: std::vector files; for (const auto directory : _paths) { + log_verbose("FileIndex:Scanning for %s in '%s'", _pattern.c_str(), directory.c_str()); + auto pattern = Path::Combine(directory, _pattern); auto scanner = Path::ScanDirectory(pattern, true); while (scanner->Next()) @@ -161,6 +177,7 @@ private: std::vector items; try { + log_verbose("FileIndex:Loading index: '%s'", _indexPath.c_str()); auto fs = FileStream(_indexPath, FILE_MODE_OPEN); // Read header, check if we need to re-scan @@ -181,10 +198,14 @@ private: } loadedItems = true; } + else + { + Console::WriteLine("Index out of date, rebuilding '%s'.", _indexPath.c_str()); + } } catch (const Exception &) { - Console::Error::WriteLine("Unable to load index."); + Console::Error::WriteLine("Unable to load index: '%s'.", _indexPath.c_str()); } return std::make_tuple(loadedItems, items); } @@ -193,6 +214,7 @@ private: { try { + log_verbose("FileIndex:Writing index: '%s'", _indexPath.c_str()); auto fs = FileStream(_indexPath, FILE_MODE_WRITE); // Write header @@ -213,7 +235,7 @@ private: } catch (const Exception &) { - Console::Error::WriteLine("Unable to save index."); + Console::Error::WriteLine("Unable to save index: '%s'.", _indexPath.c_str()); } } From ff6f257a87b185931a2abab6f9e9fd57ca7a5501 Mon Sep 17 00:00:00 2001 From: Ted John Date: Wed, 30 Aug 2017 19:11:39 +0100 Subject: [PATCH 06/16] Clean up mega park import --- src/openrct2/scenario/ScenarioRepository.cpp | 75 +++++++++++--------- 1 file changed, 40 insertions(+), 35 deletions(-) diff --git a/src/openrct2/scenario/ScenarioRepository.cpp b/src/openrct2/scenario/ScenarioRepository.cpp index af3b5e5c67..d2938d0ab7 100644 --- a/src/openrct2/scenario/ScenarioRepository.cpp +++ b/src/openrct2/scenario/ScenarioRepository.cpp @@ -299,19 +299,18 @@ private: class ScenarioRepository final : public IScenarioRepository { private: - static constexpr const utf8 * SC_FILE_PATTERN = "*.sc4;*.sc6"; static constexpr uint32 HighscoreFileVersion = 1; - IPlatformEnvironment * _env; + IPlatformEnvironment * const _env; ScenarioFileIndex const _fileIndex; std::vector _scenarios; std::vector _highscores; public: ScenarioRepository(IPlatformEnvironment * env) - : _fileIndex(env) + : _env(env), + _fileIndex(env) { - _env = env; } virtual ~ScenarioRepository() @@ -321,17 +320,17 @@ public: void Scan() override { - _scenarios.clear(); + ImportMegaPark(); + // Reload scenarios from index + _scenarios.clear(); auto scenarios = _fileIndex.LoadOrBuild(); for (auto scenario : scenarios) { AddScenario(scenario); } - - // std::string mpdatdir = _env->GetFilePath(PATHID::MP_DAT); - // ConvertMegaPark(mpdatdir, openrct2dir); + // Sort the scenarios and load the highscores Sort(); LoadScores(); LoadLegacyScores(); @@ -434,38 +433,44 @@ private: return (scenario_index_entry *)repo->GetByPath(path); } - void ConvertMegaPark(std::string &mpdatDir, std::string &scenarioDir) + /** + * Mega Park from RollerCoaster Tycoon 1 is stored in an encrypted hidden file: mp.dat. + * Decrypt the file and save it as sc21.sc4 in the user's scenario directory. + */ + void ImportMegaPark() { - //Convert mp.dat from RCT1 Data directory into SC21.SC4 (Mega Park) - utf8 mpdatPath[MAX_PATH]; - utf8 sc21Path[MAX_PATH]; - - String::Set(mpdatPath, sizeof(mpdatPath), mpdatDir.c_str()); - - if (platform_file_exists(mpdatPath)) + auto mpdatPath = _env->GetFilePath(PATHID::MP_DAT); + auto scenarioDirectory = _env->GetDirectoryPath(DIRBASE::USER, DIRID::SCENARIO); + auto sc21Path = Path::Combine(scenarioDirectory, "sc21.sc4"); + if (File::Exists(mpdatPath) && !File::Exists(sc21Path)) { - //Make sure the scenario directory exists, and that SC21.SC4 hasn't already been created - String::Set(sc21Path, sizeof(sc21Path), scenarioDir.c_str()); - platform_ensure_directory_exists(sc21Path); - Path::Append(sc21Path, sizeof(sc21Path), "SC21.SC4"); - - if (!platform_file_exists(sc21Path)) { - size_t length; - auto mpdat = (uint8 *)(File::ReadAllBytes(mpdatPath, &length)); - auto outFS = FileStream(sc21Path, FILE_MODE_WRITE); - - for (uint32 i = 0; i < (uint32)length; i++) - { - //Rotate each byte of mp.dat left by 4 bits to convert - mpdat[i] = rol8(mpdat[i], 4); - } - - outFS.WriteArray(mpdat, length); - Memory::FreeArray(mpdat, length); - } + ConvertMegaPark(mpdatPath, sc21Path); } } + /** + * Converts Mega Park to normalised file location (mp.dat to sc21.sc4) + * @param Full path to mp.dat + * @param Full path to sc21.dat + */ + void ConvertMegaPark(const std::string &srcPath, const std::string &dstPath) + { + auto directory = Path::GetDirectory(dstPath); + platform_ensure_directory_exists(directory.c_str()); + + size_t length; + auto mpdat = (uint8 *)(File::ReadAllBytes(srcPath, &length)); + + // Rotate each byte of mp.dat left by 4 bits to convert + for (size_t i = 0; i < length; i++) + { + mpdat[i] = rol8(mpdat[i], 4); + } + + File::WriteAllBytes(dstPath, mpdat, length); + Memory::FreeArray(mpdat, length); + } + void AddScenario(const scenario_index_entry &entry) { auto filename = Path::GetFileName(entry.path); From ed8483f5c35e2db414e6e1d52b9a04aa1631b370 Mon Sep 17 00:00:00 2001 From: Ted John Date: Wed, 30 Aug 2017 19:27:25 +0100 Subject: [PATCH 07/16] Time index building and handle bad files --- src/openrct2/core/FileIndex.hpp | 26 ++++++++++++++------ src/openrct2/scenario/ScenarioRepository.cpp | 14 +++++++---- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/src/openrct2/core/FileIndex.hpp b/src/openrct2/core/FileIndex.hpp index 0e4f996847..ba1bc49fd0 100644 --- a/src/openrct2/core/FileIndex.hpp +++ b/src/openrct2/core/FileIndex.hpp @@ -16,6 +16,7 @@ #pragma once +#include #include #include #include @@ -62,30 +63,30 @@ private: // Index file format version which when incremented forces a rebuild static constexpr uint8 FILE_INDEX_VERSION = 4; - // Magic number for the index (to distinguish) + std::string const _name; uint32 const _magicNumber; - // Version for the specialised index uint8 const _version; - // Path to save the index at std::string const _indexPath; - // Pattern std::string const _pattern; std::vector const _paths; public: /** * Creates a new FileIndex. + * @param name Name of the index (used for logging). * @param magicNumber Magic number for the index (to distinguish between different index files). * @param version Version of the specialised index, increment this to force a rebuild. * @param indexPath Full path to read and write the index file to. * @param pattern The search pattern for indexing files. * @param paths A list of search directories. */ - FileIndex(uint32 magicNumber, + FileIndex(std::string name, + uint32 magicNumber, uint8 version, std::string indexPath, std::string pattern, std::vector paths) : + _name(name), _magicNumber(magicNumber), _version(version), _indexPath(indexPath), @@ -113,13 +114,21 @@ public: else { // Index was not loaded + Console::WriteLine("Building %s", _name.c_str()); + auto startTime = std::chrono::high_resolution_clock::now(); for (auto filePath : scanResult.Files) { log_verbose("FileIndex:Indexing '%s'", filePath.c_str()); auto item = Create(filePath); - items.push_back(item); + if (std::get<0>(item)) + { + items.push_back(std::get<1>(item)); + } } WriteIndexFile(scanResult.Stats, items); + auto endTime = std::chrono::high_resolution_clock::now(); + auto duration = (std::chrono::duration)(endTime - startTime); + Console::WriteLine("Finished building %s in %.2f seconds.", _name.c_str(), duration.count()); } return items; } @@ -127,8 +136,9 @@ public: protected: /** * Loads the given file and creates the item representing the data to store in the index. + * TODO Use std::optional when C++17 is available. */ - virtual TItem Create(const std::string &path) const abstract; + virtual std::tuple Create(const std::string &path) const abstract; /** * Serialises an index item to the given stream. @@ -200,7 +210,7 @@ private: } else { - Console::WriteLine("Index out of date, rebuilding '%s'.", _indexPath.c_str()); + Console::WriteLine("%s out of date", _name.c_str()); } } catch (const Exception &) diff --git a/src/openrct2/scenario/ScenarioRepository.cpp b/src/openrct2/scenario/ScenarioRepository.cpp index d2938d0ab7..2fc04340b1 100644 --- a/src/openrct2/scenario/ScenarioRepository.cpp +++ b/src/openrct2/scenario/ScenarioRepository.cpp @@ -146,7 +146,8 @@ private: public: ScenarioFileIndex(IPlatformEnvironment * env) : - FileIndex(MAGIC_NUMBER, + FileIndex("scenario index", + MAGIC_NUMBER, VERSION, env->GetFilePath(PATHID::CACHE_SCENARIOS), std::string(PATTERN), @@ -158,15 +159,18 @@ public: } protected: - scenario_index_entry Create(const std::string &path) const override + std::tuple Create(const std::string &path) const override { scenario_index_entry entry; auto timestamp = File::GetLastModified(path); - if (!GetScenarioInfo(path, timestamp, &entry)) + if (GetScenarioInfo(path, timestamp, &entry)) { - // TODO + return std::make_tuple(true, entry); + } + else + { + return std::make_tuple(true, scenario_index_entry()); } - return entry; } void Serialise(IStream * stream, const scenario_index_entry &item) const override From a118b09e7923e57749bff790d9b034c4a9379bcb Mon Sep 17 00:00:00 2001 From: Ted John Date: Wed, 30 Aug 2017 19:32:28 +0100 Subject: [PATCH 08/16] Rebuild index if language has changed --- src/openrct2/core/FileIndex.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/openrct2/core/FileIndex.hpp b/src/openrct2/core/FileIndex.hpp index ba1bc49fd0..b7b122dd77 100644 --- a/src/openrct2/core/FileIndex.hpp +++ b/src/openrct2/core/FileIndex.hpp @@ -195,6 +195,7 @@ private: if (header.MagicNumber == _magicNumber && header.VersionA == FILE_INDEX_VERSION && header.VersionB == _version && + header.LanguageId == gCurrentLanguage && header.Stats.TotalFiles == stats.TotalFiles && header.Stats.TotalFileSize == stats.TotalFileSize && header.Stats.FileDateModifiedChecksum == stats.FileDateModifiedChecksum && From 92b17b149c5f65856cc6c6efa48c2c527a64f0dc Mon Sep 17 00:00:00 2001 From: Ted John Date: Wed, 30 Aug 2017 19:35:43 +0100 Subject: [PATCH 09/16] Remove old index code from ScenarioRepository --- src/openrct2/scenario/ScenarioRepository.cpp | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/openrct2/scenario/ScenarioRepository.cpp b/src/openrct2/scenario/ScenarioRepository.cpp index 2fc04340b1..8fb1f85a61 100644 --- a/src/openrct2/scenario/ScenarioRepository.cpp +++ b/src/openrct2/scenario/ScenarioRepository.cpp @@ -20,7 +20,6 @@ #include "../core/Console.hpp" #include "../core/File.h" #include "../core/FileIndex.hpp" -#include "../core/FileScanner.h" #include "../core/FileStream.hpp" #include "../core/Math.hpp" #include "../core/Path.hpp" @@ -42,20 +41,6 @@ extern "C" using namespace OpenRCT2; -constexpr uint32 SCENARIO_REPOSITORY_MAGIC_NUMBER = 0x58444953; -constexpr uint16 SCENARIO_REPOSITORY_VERSION = 1; - -struct ScenarioRepositoryHeader -{ - uint32 MagicNumber; - uint16 Version; - uint32 TotalFiles; - uint64 TotalFileSize; - uint32 FileDateModifiedChecksum; - uint32 PathChecksum; - uint32 NumItems; -}; - static sint32 ScenarioCategoryCompare(sint32 categoryA, sint32 categoryB) { if (categoryA == categoryB) return 0; From 8808444562268d03baa4705b3eb396eccf812f8a Mon Sep 17 00:00:00 2001 From: Ted John Date: Wed, 30 Aug 2017 21:01:07 +0100 Subject: [PATCH 10/16] Refactor TrackDesignRepository to use FileIndex --- src/openrct2/core/FileIndex.hpp | 8 +- src/openrct2/core/String.cpp | 5 + src/openrct2/core/String.hpp | 1 + src/openrct2/ride/TrackDesignRepository.cpp | 251 +++++++------------ src/openrct2/scenario/ScenarioRepository.cpp | 2 +- 5 files changed, 106 insertions(+), 161 deletions(-) diff --git a/src/openrct2/core/FileIndex.hpp b/src/openrct2/core/FileIndex.hpp index b7b122dd77..736d7a01f3 100644 --- a/src/openrct2/core/FileIndex.hpp +++ b/src/openrct2/core/FileIndex.hpp @@ -68,7 +68,9 @@ private: uint8 const _version; std::string const _indexPath; std::string const _pattern; - std::vector const _paths; + +public: + std::vector const SearchPaths; public: /** @@ -91,7 +93,7 @@ public: _version(version), _indexPath(indexPath), _pattern(pattern), - _paths(paths) + SearchPaths(paths) { } @@ -155,7 +157,7 @@ private: { DirectoryStats stats; std::vector files; - for (const auto directory : _paths) + for (const auto directory : SearchPaths) { log_verbose("FileIndex:Scanning for %s in '%s'", _pattern.c_str(), directory.c_str()); diff --git a/src/openrct2/core/String.cpp b/src/openrct2/core/String.cpp index 650bbfca28..febc475c10 100644 --- a/src/openrct2/core/String.cpp +++ b/src/openrct2/core/String.cpp @@ -142,6 +142,11 @@ namespace String } } + bool StartsWith(const std::string &str, const std::string &match, bool ignoreCase) + { + return StartsWith(str.c_str(), match.c_str(), ignoreCase); + } + size_t IndexOf(const utf8 * str, utf8 match, size_t startIndex) { const utf8 * ch = str + startIndex; diff --git a/src/openrct2/core/String.hpp b/src/openrct2/core/String.hpp index 97d43173df..74613b02a9 100644 --- a/src/openrct2/core/String.hpp +++ b/src/openrct2/core/String.hpp @@ -36,6 +36,7 @@ namespace String bool Equals(const std::string &a, const std::string &b, bool ignoreCase = false); bool Equals(const utf8 * a, const utf8 * b, bool ignoreCase = false); bool StartsWith(const utf8 * str, const utf8 * match, bool ignoreCase = false); + bool StartsWith(const std::string &str, const std::string &match, bool ignoreCase = false); size_t IndexOf(const utf8 * str, utf8 match, size_t startIndex = 0); size_t LastIndexOf(const utf8 * str, utf8 match); diff --git a/src/openrct2/ride/TrackDesignRepository.cpp b/src/openrct2/ride/TrackDesignRepository.cpp index 944559417e..c662b69618 100644 --- a/src/openrct2/ride/TrackDesignRepository.cpp +++ b/src/openrct2/ride/TrackDesignRepository.cpp @@ -21,14 +21,14 @@ #include "../core/Collections.hpp" #include "../core/Console.hpp" #include "../core/File.h" -#include "../core/FileScanner.h" +#include "../core/FileIndex.hpp" #include "../core/FileStream.hpp" #include "../core/Path.hpp" -#include "RideGroupManager.h" #include "../core/String.hpp" #include "../object/ObjectRepository.h" #include "../object/RideObject.h" #include "../PlatformEnvironment.h" +#include "RideGroupManager.h" #include "TrackDesignRepository.h" extern "C" @@ -38,52 +38,107 @@ extern "C" using namespace OpenRCT2; -#pragma pack(push, 1) -struct TrackRepositoryHeader -{ - uint32 MagicNumber; - uint16 Version; - uint32 TotalFiles; - uint64 TotalFileSize; - uint32 FileDateModifiedChecksum; - uint32 PathChecksum; - uint32 NumItems; -}; -#pragma pack(pop) - struct TrackRepositoryItem { std::string Name; std::string Path; uint8 RideType = 0; std::string ObjectEntry; - uint32 Flags; + uint32 Flags = 0; }; -constexpr uint32 TRACK_REPOSITORY_MAGIC_NUMBER = 0x58444954; -constexpr uint16 TRACK_REPOSITORY_VERSION = 1; - enum TRACK_REPO_ITEM_FLAGS { TRIF_READ_ONLY = (1 << 0), }; +static std::string GetNameFromTrackPath(const std::string &path) +{ + std::string name = Path::GetFileNameWithoutExtension(path); + //The track name should be the file name until the first instance of a dot + name = name.substr(0, name.find_first_of(".")); + return name; +} + +class TrackDesignFileIndex final : public FileIndex +{ +private: + static constexpr uint32 MAGIC_NUMBER = 0x58444954; // TIDX + static constexpr uint16 VERSION = 1; + static constexpr auto PATTERN = "*.td4;*.td6"; + +public: + TrackDesignFileIndex(IPlatformEnvironment * env) : + FileIndex("track design index", + MAGIC_NUMBER, + VERSION, + env->GetFilePath(PATHID::CACHE_TRACKS), + std::string(PATTERN), + std::vector({ + env->GetDirectoryPath(DIRBASE::RCT1, DIRID::TRACK), + env->GetDirectoryPath(DIRBASE::RCT2, DIRID::TRACK), + env->GetDirectoryPath(DIRBASE::USER, DIRID::TRACK) })) + { + } + +public: + std::tuple Create(const std::string &path) const override + { + auto td6 = track_design_open(path.c_str()); + if (td6 != nullptr) + { + TrackRepositoryItem item; + item.Name = GetNameFromTrackPath(path); + item.Path = path; + item.RideType = td6->type; + item.ObjectEntry = std::string(td6->vehicle_object.name, 8); + item.Flags = 0; + if (IsTrackReadOnly(path)) + { + item.Flags |= TRIF_READ_ONLY; + } + track_design_dispose(td6); + return std::make_tuple(true, item); + } + else + { + return std::make_tuple(true, TrackRepositoryItem()); + } + } + +protected: + void Serialise(IStream * stream, const TrackRepositoryItem &item) const override + { + stream->WriteValue(item); + } + + TrackRepositoryItem Deserialise(IStream * stream) const override + { + return stream->ReadValue(); + } + +private: + bool IsTrackReadOnly(const std::string &path) const + { + return + String::StartsWith(path, SearchPaths[0]) || + String::StartsWith(path, SearchPaths[1]); + } +}; + class TrackDesignRepository final : public ITrackDesignRepository { private: - static constexpr const utf8 * TD_FILE_PATTERN = "*.td4;*.td6"; - - IPlatformEnvironment * _env; - + IPlatformEnvironment * const _env; + TrackDesignFileIndex const _fileIndex; std::vector _items; - QueryDirectoryResult _directoryQueryResult = { 0 }; public: TrackDesignRepository(IPlatformEnvironment * env) + : _env(env), + _fileIndex(env) { Guard::ArgumentNotNull(env); - - _env = env; } virtual ~TrackDesignRepository() final @@ -227,21 +282,14 @@ public: void Scan() override { - std::string rct2Directory = _env->GetDirectoryPath(DIRBASE::RCT2, DIRID::TRACK); - std::string userDirectory = _env->GetDirectoryPath(DIRBASE::USER, DIRID::TRACK); - _items.clear(); - _directoryQueryResult = { 0 }; - Query(rct2Directory); - Query(userDirectory); - - if (!Load()) + auto trackDesigns = _fileIndex.LoadOrBuild(); + for (auto td : trackDesigns) { - Scan(rct2Directory, TRIF_READ_ONLY); - Scan(userDirectory); - SortItems(); - Save(); + _items.push_back(td); } + + SortItems(); } bool Delete(const std::string &path) override @@ -295,48 +343,18 @@ public: std::string newPath = Path::Combine(installDir, fileName); if (File::Copy(path, newPath, false)) { - AddTrack(path); - SortItems(); - result = path; + auto td = _fileIndex.Create(path); + if (std::get<0>(td)) + { + _items.push_back(std::get<1>(td)); + SortItems(); + result = path; + } } return result; } private: - void Query(const std::string &directory) - { - std::string pattern = Path::Combine(directory, TD_FILE_PATTERN); - Path::QueryDirectory(&_directoryQueryResult, pattern); - } - - void Scan(const std::string &directory, uint32 flags = 0) - { - std::string pattern = Path::Combine(directory, TD_FILE_PATTERN); - IFileScanner * scanner = Path::ScanDirectory(pattern, true); - while (scanner->Next()) - { - const utf8 * path = scanner->GetPath(); - AddTrack(path, flags); - } - delete scanner; - } - - void AddTrack(const std::string path, uint32 flags = 0) - { - rct_track_td6 * td6 = track_design_open(path.c_str()); - if (td6 != nullptr) - { - TrackRepositoryItem item; - item.Name = GetNameFromTrackPath(path); - item.Path = path; - item.RideType = td6->type; - item.ObjectEntry = std::string(td6->vehicle_object.name, 8); - item.Flags = flags; - _items.push_back(item); - track_design_dispose(td6); - } - } - void SortItems() { std::sort(_items.begin(), _items.end(), [](const TrackRepositoryItem &a, @@ -350,78 +368,6 @@ private: }); } - bool Load() - { - std::string path = _env->GetFilePath(PATHID::CACHE_TRACKS); - bool result = false; - try - { - auto fs = FileStream(path, FILE_MODE_OPEN); - - // Read header, check if we need to re-scan - auto header = fs.ReadValue(); - if (header.MagicNumber == TRACK_REPOSITORY_MAGIC_NUMBER && - header.Version == TRACK_REPOSITORY_VERSION && - header.TotalFiles == _directoryQueryResult.TotalFiles && - header.TotalFileSize == _directoryQueryResult.TotalFileSize && - header.FileDateModifiedChecksum == _directoryQueryResult.FileDateModifiedChecksum && - header.PathChecksum == _directoryQueryResult.PathChecksum) - { - // Directory is the same, just read the saved items - for (uint32 i = 0; i < header.NumItems; i++) - { - TrackRepositoryItem item; - item.Name = fs.ReadStdString(); - item.Path = fs.ReadStdString(); - item.RideType = fs.ReadValue(); - item.ObjectEntry = fs.ReadStdString(); - item.Flags = fs.ReadValue(); - _items.push_back(item); - } - result = true; - } - } - catch (const Exception &) - { - Console::Error::WriteLine("Unable to write object repository index."); - } - return result; - } - - void Save() const - { - std::string path = _env->GetFilePath(PATHID::CACHE_TRACKS); - try - { - auto fs = FileStream(path, FILE_MODE_WRITE); - - // Write header - TrackRepositoryHeader header = { 0 }; - header.MagicNumber = TRACK_REPOSITORY_MAGIC_NUMBER; - header.Version = TRACK_REPOSITORY_VERSION; - header.TotalFiles = _directoryQueryResult.TotalFiles; - header.TotalFileSize = _directoryQueryResult.TotalFileSize; - header.FileDateModifiedChecksum = _directoryQueryResult.FileDateModifiedChecksum; - header.PathChecksum = _directoryQueryResult.PathChecksum; - header.NumItems = (uint32)_items.size(); - fs.WriteValue(header); - - // Write items - for (const auto item : _items) - { - fs.WriteString(item.Name); - fs.WriteString(item.Path); - fs.WriteValue(item.RideType); - fs.WriteString(item.ObjectEntry); - fs.WriteValue(item.Flags); - } - } - catch (const Exception &) - { - Console::Error::WriteLine("Unable to write object repository index."); - } - } - size_t GetTrackIndex(const std::string &path) const { for (size_t i = 0; i < _items.size(); i++) @@ -444,15 +390,6 @@ private: } return result; } - -public: - static std::string GetNameFromTrackPath(const std::string &path) - { - std::string name = Path::GetFileNameWithoutExtension(path); - //The track name should be the file name until the first instance of a dot - name = name.substr(0, name.find_first_of(".")); - return name; - } }; static TrackDesignRepository * _trackDesignRepository = nullptr; @@ -522,6 +459,6 @@ extern "C" utf8 * track_repository_get_name_from_path(const utf8 * path) { - return String::Duplicate(TrackDesignRepository::GetNameFromTrackPath(path)); + return String::Duplicate(GetNameFromTrackPath(path)); } } diff --git a/src/openrct2/scenario/ScenarioRepository.cpp b/src/openrct2/scenario/ScenarioRepository.cpp index 8fb1f85a61..dc52dd8e6e 100644 --- a/src/openrct2/scenario/ScenarioRepository.cpp +++ b/src/openrct2/scenario/ScenarioRepository.cpp @@ -125,7 +125,7 @@ static void scenario_highscore_free(scenario_highscore_entry * highscore) class ScenarioFileIndex final : public FileIndex { private: - static constexpr uint32 MAGIC_NUMBER = 0x58444953; + static constexpr uint32 MAGIC_NUMBER = 0x58444953; // SIDX static constexpr uint16 VERSION = 1; static constexpr auto PATTERN = "*.sc4;*.sc6"; From f2cd0b2f3b0bcecb342ce40eabcb18b55365a56e Mon Sep 17 00:00:00 2001 From: Ted John Date: Wed, 30 Aug 2017 21:58:43 +0100 Subject: [PATCH 11/16] Refactor ObjectRepository to use FileIndex --- src/openrct2/core/FileIndex.hpp | 47 ++- src/openrct2/object/ObjectManager.cpp | 2 +- src/openrct2/object/ObjectRepository.cpp | 389 ++++++++------------ src/openrct2/ride/TrackDesignRepository.cpp | 14 +- 4 files changed, 196 insertions(+), 256 deletions(-) diff --git a/src/openrct2/core/FileIndex.hpp b/src/openrct2/core/FileIndex.hpp index 736d7a01f3..dbc4b19659 100644 --- a/src/openrct2/core/FileIndex.hpp +++ b/src/openrct2/core/FileIndex.hpp @@ -116,25 +116,18 @@ public: else { // Index was not loaded - Console::WriteLine("Building %s", _name.c_str()); - auto startTime = std::chrono::high_resolution_clock::now(); - for (auto filePath : scanResult.Files) - { - log_verbose("FileIndex:Indexing '%s'", filePath.c_str()); - auto item = Create(filePath); - if (std::get<0>(item)) - { - items.push_back(std::get<1>(item)); - } - } - WriteIndexFile(scanResult.Stats, items); - auto endTime = std::chrono::high_resolution_clock::now(); - auto duration = (std::chrono::duration)(endTime - startTime); - Console::WriteLine("Finished building %s in %.2f seconds.", _name.c_str(), duration.count()); + items = Build(scanResult); } return items; } + std::vector Rebuild() const + { + auto scanResult = Scan(); + auto items = Build(scanResult); + return items; + } + protected: /** * Loads the given file and creates the item representing the data to store in the index. @@ -183,6 +176,30 @@ private: return ScanResult(stats, files); } + std::vector Build(const ScanResult &scanResult) const + { + std::vector items; + Console::WriteLine("Building %s (%zu items)", _name.c_str(), scanResult.Files.size()); + + auto startTime = std::chrono::high_resolution_clock::now(); + for (auto filePath : scanResult.Files) + { + log_verbose("FileIndex:Indexing '%s'", filePath.c_str()); + auto item = Create(filePath); + if (std::get<0>(item)) + { + items.push_back(std::get<1>(item)); + } + } + + WriteIndexFile(scanResult.Stats, items); + + auto endTime = std::chrono::high_resolution_clock::now(); + auto duration = (std::chrono::duration)(endTime - startTime); + Console::WriteLine("Finished building %s in %.2f seconds.", _name.c_str(), duration.count()); + return items; + } + std::tuple> ReadIndexFile(const DirectoryStats &stats) const { bool loadedItems = false; diff --git a/src/openrct2/object/ObjectManager.cpp b/src/openrct2/object/ObjectManager.cpp index 0995316594..f4885539bd 100644 --- a/src/openrct2/object/ObjectManager.cpp +++ b/src/openrct2/object/ObjectManager.cpp @@ -581,7 +581,7 @@ private: return loadedObject; } - void ReportMissingObject(const rct_object_entry * entry) + static void ReportMissingObject(const rct_object_entry * entry) { utf8 objName[9] = { 0 }; Memory::Copy(objName, entry->name, 8); diff --git a/src/openrct2/object/ObjectRepository.cpp b/src/openrct2/object/ObjectRepository.cpp index ce262302fe..69da09c48b 100644 --- a/src/openrct2/object/ObjectRepository.cpp +++ b/src/openrct2/object/ObjectRepository.cpp @@ -15,14 +15,13 @@ #pragma endregion #include -#include #include #include #include #include "../common.h" #include "../core/Console.hpp" -#include "../core/FileScanner.h" +#include "../core/FileIndex.hpp" #include "../core/FileStream.hpp" #include "../core/Guard.hpp" #include "../core/IStream.hpp" @@ -54,22 +53,6 @@ extern "C" using namespace OpenRCT2; -constexpr uint16 OBJECT_REPOSITORY_VERSION = 11; - -#pragma pack(push, 1) -struct ObjectRepositoryHeader -{ - uint16 Version; - uint16 LanguageId; - uint32 TotalFiles; - uint64 TotalFileSize; - uint32 FileDateModifiedChecksum; - uint32 PathChecksum; - uint32 NumItems; -}; -assert_struct_size(ObjectRepositoryHeader, 28); -#pragma pack(pop) - struct ObjectEntryHash { size_t operator()(const rct_object_entry &entry) const @@ -95,17 +78,131 @@ using ObjectEntryMap = std::unordered_map +{ +private: + static constexpr uint32 MAGIC_NUMBER = 0x5844494F; // OIDX + static constexpr uint16 VERSION = 15; + static constexpr auto PATTERN = "*.dat"; + +public: + ObjectFileIndex(IPlatformEnvironment * env) : + FileIndex("object index", + MAGIC_NUMBER, + VERSION, + env->GetFilePath(PATHID::CACHE_OBJECTS), + std::string(PATTERN), + std::vector({ + env->GetDirectoryPath(DIRBASE::RCT2, DIRID::OBJECT), + env->GetDirectoryPath(DIRBASE::USER, DIRID::OBJECT) })) + { + } + +public: + std::tuple Create(const std::string &path) const override + { + auto object = ObjectFactory::CreateObjectFromLegacyFile(path.c_str()); + if (object != nullptr) + { + ObjectRepositoryItem item = { 0 }; + item.ObjectEntry = *object->GetObjectEntry(); + item.Path = String::Duplicate(path); + item.Name = String::Duplicate(object->GetName()); + object->SetRepositoryItem(&item); + delete object; + return std::make_tuple(true, item); + } + else + { + return std::make_tuple(true, ObjectRepositoryItem()); + } + } + +protected: + void Serialise(IStream * stream, const ObjectRepositoryItem &item) const override + { + stream->WriteValue(item.ObjectEntry); + stream->WriteString(item.Path); + stream->WriteString(item.Name); + + switch (item.ObjectEntry.flags & 0x0F) { + case OBJECT_TYPE_RIDE: + stream->WriteValue(item.RideFlags); + for (sint32 i = 0; i < 2; i++) + { + stream->WriteValue(item.RideCategory[i]); + } + for (sint32 i = 0; i < MAX_RIDE_TYPES_PER_RIDE_ENTRY; i++) + { + stream->WriteValue(item.RideType[i]); + } + stream->WriteValue(item.RideGroupIndex); + break; + case OBJECT_TYPE_SCENERY_SETS: + stream->WriteValue(item.NumThemeObjects); + for (uint16 i = 0; i < item.NumThemeObjects; i++) + { + stream->WriteValue(item.ThemeObjects[i]); + } + break; + } + } + + ObjectRepositoryItem Deserialise(IStream * stream) const override + { + ObjectRepositoryItem item = { 0 }; + + item.ObjectEntry = stream->ReadValue(); + item.Path = stream->ReadString(); + item.Name = stream->ReadString(); + + switch (item.ObjectEntry.flags & 0x0F) { + case OBJECT_TYPE_RIDE: + item.RideFlags = stream->ReadValue(); + for (sint32 i = 0; i < 2; i++) + { + item.RideCategory[i] = stream->ReadValue(); + } + for (sint32 i = 0; i < MAX_RIDE_TYPES_PER_RIDE_ENTRY; i++) + { + item.RideType[i] = stream->ReadValue(); + } + item.RideGroupIndex = stream->ReadValue(); + break; + case OBJECT_TYPE_SCENERY_SETS: + item.NumThemeObjects = stream->ReadValue(); + item.ThemeObjects = Memory::AllocateArray(item.NumThemeObjects); + for (uint16 i = 0; i < item.NumThemeObjects; i++) + { + item.ThemeObjects[i] = stream->ReadValue(); + } + break; + } + return item; + } + +private: + bool IsTrackReadOnly(const std::string &path) const + { + return + String::StartsWith(path, SearchPaths[0]) || + String::StartsWith(path, SearchPaths[1]); + } +}; + class ObjectRepository final : public IObjectRepository { - const IPlatformEnvironment * _env = nullptr; + IPlatformEnvironment * const _env = nullptr; + ObjectFileIndex const _fileIndex; std::vector _items; - QueryDirectoryResult _queryDirectoryResult = { 0 }; ObjectEntryMap _itemMap; uint16 _languageId = 0; sint32 _numConflicts = 0; public: - ObjectRepository(IPlatformEnvironment * env) : _env(env) + ObjectRepository(IPlatformEnvironment * env) + : _env(env), + _fileIndex(env) { } @@ -117,24 +214,17 @@ public: void LoadOrConstruct() override { ClearItems(); - - Query(); - if (!Load()) - { - _languageId = gCurrentLanguage; - Scan(); - Save(); - } - - // SortItems(); + auto items = _fileIndex.LoadOrBuild(); + AddItems(items); + SortItems(); } void Construct() override { _languageId = gCurrentLanguage; - Query(); - Scan(); - Save(); + auto items = _fileIndex.Rebuild(); + AddItems(items); + SortItems(); } size_t GetNumObjects() const override @@ -274,146 +364,6 @@ private: _itemMap.clear(); } - void Query() - { - _queryDirectoryResult = { 0 }; - - const std::string &rct2Path = _env->GetDirectoryPath(DIRBASE::RCT2, DIRID::OBJECT); - const std::string &openrct2Path = _env->GetDirectoryPath(DIRBASE::USER, DIRID::OBJECT); - QueryDirectory(&_queryDirectoryResult, rct2Path); - QueryDirectory(&_queryDirectoryResult, openrct2Path); - } - - void QueryDirectory(QueryDirectoryResult * result, const std::string &directory) - { - utf8 pattern[MAX_PATH]; - String::Set(pattern, sizeof(pattern), directory.c_str()); - Path::Append(pattern, sizeof(pattern), "*.dat"); - Path::QueryDirectory(result, pattern); - } - - void Scan() - { - Console::WriteLine("Scanning %lu objects...", _queryDirectoryResult.TotalFiles); - _numConflicts = 0; - - auto startTime = std::chrono::high_resolution_clock::now(); - - const std::string &rct2Path = _env->GetDirectoryPath(DIRBASE::RCT2, DIRID::OBJECT); - const std::string &openrct2Path = _env->GetDirectoryPath(DIRBASE::USER, DIRID::OBJECT); - ScanDirectory(rct2Path); - ScanDirectory(openrct2Path); - - auto endTime = std::chrono::high_resolution_clock::now(); - std::chrono::duration duration = endTime - startTime; - - Console::WriteLine("Scanning complete in %.2f seconds.", duration.count()); - if (_numConflicts > 0) - { - Console::WriteLine("%d object conflicts found.", _numConflicts); - } - } - - void ScanDirectory(const std::string &directory) - { - utf8 pattern[MAX_PATH]; - String::Set(pattern, sizeof(pattern), directory.c_str()); - Path::Append(pattern, sizeof(pattern), "*.dat"); - - IFileScanner * scanner = Path::ScanDirectory(pattern, true); - while (scanner->Next()) - { - const utf8 * enumPath = scanner->GetPath(); - ScanObject(enumPath); - } - delete scanner; - } - - void ScanObject(const utf8 * path) - { - Object * object = ObjectFactory::CreateObjectFromLegacyFile(path); - if (object != nullptr) - { - ObjectRepositoryItem item = { 0 }; - item.ObjectEntry = *object->GetObjectEntry(); - item.Path = String::Duplicate(path); - item.Name = String::Duplicate(object->GetName()); - object->SetRepositoryItem(&item); - AddItem(&item); - - delete object; - } - } - - bool Load() - { - const std::string &path = _env->GetFilePath(PATHID::CACHE_OBJECTS); - try - { - auto fs = FileStream(path, FILE_MODE_OPEN); - auto header = fs.ReadValue(); - - if (header.Version == OBJECT_REPOSITORY_VERSION && - header.LanguageId == gCurrentLanguage && - header.TotalFiles == _queryDirectoryResult.TotalFiles && - header.TotalFileSize == _queryDirectoryResult.TotalFileSize && - header.FileDateModifiedChecksum == _queryDirectoryResult.FileDateModifiedChecksum && - header.PathChecksum == _queryDirectoryResult.PathChecksum) - { - // Header matches, so the index is not out of date - - // Buffer the rest of file into memory to speed up item reading - size_t dataSize = (size_t)(fs.GetLength() - fs.GetPosition()); - void * data = fs.ReadArray(dataSize); - auto ms = MemoryStream(data, dataSize, MEMORY_ACCESS::READ | MEMORY_ACCESS::OWNER); - - // Read items - for (uint32 i = 0; i < header.NumItems; i++) - { - ObjectRepositoryItem item = ReadItem(&ms); - AddItem(&item); - } - return true; - } - Console::WriteLine("Object repository is out of date."); - return false; - } - catch (const IOException &) - { - return false; - } - } - - void Save() const - { - const std::string &path = _env->GetFilePath(PATHID::CACHE_OBJECTS); - try - { - auto fs = FileStream(path, FILE_MODE_WRITE); - - // Write header - ObjectRepositoryHeader header; - header.Version = OBJECT_REPOSITORY_VERSION; - header.LanguageId = _languageId; - header.TotalFiles = _queryDirectoryResult.TotalFiles; - header.TotalFileSize = _queryDirectoryResult.TotalFileSize; - header.FileDateModifiedChecksum = _queryDirectoryResult.FileDateModifiedChecksum; - header.PathChecksum = _queryDirectoryResult.PathChecksum; - header.NumItems = (uint32)_items.size(); - fs.WriteValue(header); - - // Write items - for (uint32 i = 0; i < header.NumItems; i++) - { - WriteItem(&fs, _items[i]); - } - } - catch (const IOException &) - { - log_error("Unable to write object repository index to '%s'.", path.c_str()); - } - } - void SortItems() { std::sort(_items.begin(), _items.end(), [](const ObjectRepositoryItem &a, @@ -422,6 +372,12 @@ private: return strcmp(a.Name, b.Name) < 0; }); + // Fix the IDs + for (size_t i = 0; i < _items.size(); i++) + { + _items[i].Id = i; + } + // Rebuild item map _itemMap.clear(); for (size_t i = 0; i < _items.size(); i++) @@ -431,85 +387,42 @@ private: } } - bool AddItem(ObjectRepositoryItem * item) + void AddItems(const std::vector &items) { - const ObjectRepositoryItem * conflict = FindObject(&item->ObjectEntry); + for (auto item : items) + { + AddItem(item); + } + } + + bool AddItem(const ObjectRepositoryItem &item) + { + auto conflict = FindObject(&item.ObjectEntry); if (conflict == nullptr) { size_t index = _items.size(); - item->Id = index; - _items.push_back(*item); - _itemMap[item->ObjectEntry] = index; + auto copy = item; + copy.Id = index; + _items.push_back(copy); + _itemMap[item.ObjectEntry] = index; return true; } else { _numConflicts++; Console::Error::WriteLine("Object conflict: '%s'", conflict->Path); - Console::Error::WriteLine(" : '%s'", item->Path); + Console::Error::WriteLine(" : '%s'", item.Path); return false; } } - static ObjectRepositoryItem ReadItem(IStream * stream) + void ScanObject(const std::string &path) { - ObjectRepositoryItem item = { 0 }; - - item.ObjectEntry = stream->ReadValue(); - item.Path = stream->ReadString(); - item.Name = stream->ReadString(); - - switch (item.ObjectEntry.flags & 0x0F) { - case OBJECT_TYPE_RIDE: - item.RideFlags = stream->ReadValue(); - for (sint32 i = 0; i < 2; i++) - { - item.RideCategory[i] = stream->ReadValue(); - } - for (sint32 i = 0; i < MAX_RIDE_TYPES_PER_RIDE_ENTRY; i++) - { - item.RideType[i] = stream->ReadValue(); - } - item.RideGroupIndex = stream->ReadValue(); - break; - case OBJECT_TYPE_SCENERY_SETS: - item.NumThemeObjects = stream->ReadValue(); - item.ThemeObjects = Memory::AllocateArray(item.NumThemeObjects); - for (uint16 i = 0; i < item.NumThemeObjects; i++) - { - item.ThemeObjects[i] = stream->ReadValue(); - } - break; - } - return item; - } - - static void WriteItem(IStream * stream, const ObjectRepositoryItem &item) - { - stream->WriteValue(item.ObjectEntry); - stream->WriteString(item.Path); - stream->WriteString(item.Name); - - switch (item.ObjectEntry.flags & 0x0F) { - case OBJECT_TYPE_RIDE: - stream->WriteValue(item.RideFlags); - for (sint32 i = 0; i < 2; i++) - { - stream->WriteValue(item.RideCategory[i]); - } - for (sint32 i = 0; i < MAX_RIDE_TYPES_PER_RIDE_ENTRY; i++) - { - stream->WriteValue(item.RideType[i]); - } - stream->WriteValue(item.RideGroupIndex); - break; - case OBJECT_TYPE_SCENERY_SETS: - stream->WriteValue(item.NumThemeObjects); - for (uint16 i = 0; i < item.NumThemeObjects; i++) - { - stream->WriteValue(item.ThemeObjects[i]); - } - break; + auto result = _fileIndex.Create(path); + if (std::get<0>(result)) + { + auto ori = std::get<1>(result); + AddItem(ori); } } diff --git a/src/openrct2/ride/TrackDesignRepository.cpp b/src/openrct2/ride/TrackDesignRepository.cpp index c662b69618..0ea0433bb0 100644 --- a/src/openrct2/ride/TrackDesignRepository.cpp +++ b/src/openrct2/ride/TrackDesignRepository.cpp @@ -109,12 +109,22 @@ public: protected: void Serialise(IStream * stream, const TrackRepositoryItem &item) const override { - stream->WriteValue(item); + stream->WriteString(item.Name); + stream->WriteString(item.Path); + stream->WriteValue(item.RideType); + stream->WriteString(item.ObjectEntry); + stream->WriteValue(item.Flags); } TrackRepositoryItem Deserialise(IStream * stream) const override { - return stream->ReadValue(); + TrackRepositoryItem item; + item.Name = stream->ReadStdString(); + item.Path = stream->ReadStdString(); + item.RideType = stream->ReadValue(); + item.ObjectEntry = stream->ReadStdString(); + item.Flags = stream->ReadValue(); + return item; } private: From f371d5c65feb967470111cad23876b53c6433b37 Mon Sep 17 00:00:00 2001 From: Ted John Date: Wed, 30 Aug 2017 22:05:12 +0100 Subject: [PATCH 12/16] Remove unnecessary fields in ObjectRepository --- src/openrct2/object/ObjectRepository.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/openrct2/object/ObjectRepository.cpp b/src/openrct2/object/ObjectRepository.cpp index 69da09c48b..ad9b2a5de5 100644 --- a/src/openrct2/object/ObjectRepository.cpp +++ b/src/openrct2/object/ObjectRepository.cpp @@ -196,8 +196,6 @@ class ObjectRepository final : public IObjectRepository ObjectFileIndex const _fileIndex; std::vector _items; ObjectEntryMap _itemMap; - uint16 _languageId = 0; - sint32 _numConflicts = 0; public: ObjectRepository(IPlatformEnvironment * env) @@ -221,7 +219,6 @@ public: void Construct() override { - _languageId = gCurrentLanguage; auto items = _fileIndex.Rebuild(); AddItems(items); SortItems(); @@ -389,10 +386,15 @@ private: void AddItems(const std::vector &items) { + size_t numConflicts = 0; for (auto item : items) { - AddItem(item); + if (!AddItem(item)) + { + numConflicts++; + } } + Console::Error::WriteLine("%zu object conflicts found.", numConflicts); } bool AddItem(const ObjectRepositoryItem &item) @@ -409,7 +411,6 @@ private: } else { - _numConflicts++; Console::Error::WriteLine("Object conflict: '%s'", conflict->Path); Console::Error::WriteLine(" : '%s'", item.Path); return false; From dcb78b18d9c714a00699a9d06b8c22edfd4e348c Mon Sep 17 00:00:00 2001 From: Ted John Date: Thu, 31 Aug 2017 12:17:28 +0100 Subject: [PATCH 13/16] Add HeaderSize to FileIndexHeader to increase rebuild chance when alignment changes --- src/openrct2/core/FileIndex.hpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/openrct2/core/FileIndex.hpp b/src/openrct2/core/FileIndex.hpp index dbc4b19659..36594b5ba1 100644 --- a/src/openrct2/core/FileIndex.hpp +++ b/src/openrct2/core/FileIndex.hpp @@ -52,6 +52,7 @@ private: struct FileIndexHeader { + uint32 HeaderSize = sizeof(FileIndexHeader); uint32 MagicNumber = 0; uint8 VersionA = 0; uint8 VersionB = 0; @@ -211,7 +212,8 @@ private: // Read header, check if we need to re-scan auto header = fs.ReadValue(); - if (header.MagicNumber == _magicNumber && + if (header.HeaderSize == sizeof(FileIndexHeader) && + header.MagicNumber == _magicNumber && header.VersionA == FILE_INDEX_VERSION && header.VersionB == _version && header.LanguageId == gCurrentLanguage && @@ -248,7 +250,7 @@ private: auto fs = FileStream(_indexPath, FILE_MODE_WRITE); // Write header - FileIndexHeader header = { 0 }; + FileIndexHeader header; header.MagicNumber = _magicNumber; header.VersionA = FILE_INDEX_VERSION; header.VersionB = _version; From 0df10cc41dfef2981460ef5d02b1cf68cd58aef2 Mon Sep 17 00:00:00 2001 From: Ted John Date: Thu, 31 Aug 2017 17:52:26 +0100 Subject: [PATCH 14/16] Log exception messages --- src/openrct2/core/FileIndex.hpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/openrct2/core/FileIndex.hpp b/src/openrct2/core/FileIndex.hpp index 36594b5ba1..5e9d527f45 100644 --- a/src/openrct2/core/FileIndex.hpp +++ b/src/openrct2/core/FileIndex.hpp @@ -235,9 +235,10 @@ private: Console::WriteLine("%s out of date", _name.c_str()); } } - catch (const Exception &) + catch (const std::exception &e) { Console::Error::WriteLine("Unable to load index: '%s'.", _indexPath.c_str()); + Console::Error::WriteLine("%s", e.what()); } return std::make_tuple(loadedItems, items); } @@ -265,9 +266,10 @@ private: Serialise(&fs, item); } } - catch (const Exception &) + catch (const std::exception &e) { Console::Error::WriteLine("Unable to save index: '%s'.", _indexPath.c_str()); + Console::Error::WriteLine("%s", e.what()); } } From 50735c0f68062aaf7dcac47242dd0f66bcdea5a9 Mon Sep 17 00:00:00 2001 From: Ted John Date: Thu, 31 Aug 2017 18:00:33 +0100 Subject: [PATCH 15/16] Do not log 0 conflicts --- src/openrct2/object/ObjectRepository.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/openrct2/object/ObjectRepository.cpp b/src/openrct2/object/ObjectRepository.cpp index ad9b2a5de5..3bc47ff295 100644 --- a/src/openrct2/object/ObjectRepository.cpp +++ b/src/openrct2/object/ObjectRepository.cpp @@ -394,7 +394,10 @@ private: numConflicts++; } } - Console::Error::WriteLine("%zu object conflicts found.", numConflicts); + if (numConflicts > 0) + { + Console::Error::WriteLine("%zu object conflicts found.", numConflicts); + } } bool AddItem(const ObjectRepositoryItem &item) From c8169758d3caf6b8bd573595a28d2561ab425eed Mon Sep 17 00:00:00 2001 From: Ted John Date: Thu, 31 Aug 2017 18:34:21 +0100 Subject: [PATCH 16/16] Re-implement scan-objects command line tool --- src/openrct2/cmdline/RootCommands.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/openrct2/cmdline/RootCommands.cpp b/src/openrct2/cmdline/RootCommands.cpp index 829240b60c..c7cf5498dd 100644 --- a/src/openrct2/cmdline/RootCommands.cpp +++ b/src/openrct2/cmdline/RootCommands.cpp @@ -23,6 +23,7 @@ extern "C" #include "../config/Config.h" #include "../platform/crash.h" #include "../platform/platform.h" + #include "../localisation/language.h" } #include "../core/Console.hpp" @@ -32,6 +33,7 @@ extern "C" #include "../network/network.h" #include "../object/ObjectRepository.h" #include "../OpenRCT2.h" +#include "../PlatformEnvironment.h" #include "../Version.h" #include "CommandLine.hpp" @@ -397,9 +399,13 @@ static exitcode_t HandleCommandScanObjects(CommandLineArgEnumerator * enumerator return result; } - // IPlatformEnvironment * env = OpenRCT2::SetupEnvironment(); - // IObjectRepository * objectRepository = CreateObjectRepository(env); - // objectRepository->Construct(); + auto env = OpenRCT2::CreatePlatformEnvironment(); + + // HACK: set gCurrentLanguage otherwise it be wrong for the index file + gCurrentLanguage = gConfigGeneral.language; + + auto objectRepository = CreateObjectRepository(env); + objectRepository->Construct(); return EXITCODE_OK; }