From 140a96b3a0fa652b66c701ac941cebc17ff24622 Mon Sep 17 00:00:00 2001 From: Niels Martin Hansen Date: Mon, 15 Apr 2019 19:49:30 +0200 Subject: [PATCH] Change: Limit memory allocations for each Squirrel instance This can avoid out-of-memory situations due to single scripts using up the entire address space. Instead, scripts that go above the maximum are killed. The maximum is default 1 GB per script, but can be configured by a setting. --- src/3rdparty/squirrel/squirrel/sqmem.cpp | 8 +- src/ai/ai_instance.cpp | 1 + src/ai/ai_scanner.cpp | 2 + src/lang/english.txt | 3 + src/saveload/saveload.h | 2 + src/script/api/script_object.cpp | 2 +- src/script/api/script_object.hpp | 2 + src/script/script_instance.cpp | 8 ++ src/script/script_instance.hpp | 3 + src/script/squirrel.cpp | 148 ++++++++++++++++++++++- src/script/squirrel.hpp | 30 +++++ src/settings_gui.cpp | 1 + src/settings_type.h | 1 + src/table/settings.ini | 15 +++ 14 files changed, 221 insertions(+), 5 deletions(-) diff --git a/src/3rdparty/squirrel/squirrel/sqmem.cpp b/src/3rdparty/squirrel/squirrel/sqmem.cpp index 5c1b3966aa..f4fa6309ed 100644 --- a/src/3rdparty/squirrel/squirrel/sqmem.cpp +++ b/src/3rdparty/squirrel/squirrel/sqmem.cpp @@ -9,8 +9,10 @@ #include "../../../core/alloc_func.hpp" #include "../../../safeguards.h" -void *sq_vm_malloc(SQUnsignedInteger size){ return MallocT((size_t)size); } +#ifdef SQUIRREL_DEFAULT_ALLOCATOR +void *sq_vm_malloc(SQUnsignedInteger size) { return MallocT((size_t)size); } -void *sq_vm_realloc(void *p, SQUnsignedInteger oldsize, SQUnsignedInteger size){ return ReallocT(static_cast(p), (size_t)size); } +void *sq_vm_realloc(void *p, SQUnsignedInteger oldsize, SQUnsignedInteger size) { return ReallocT(static_cast(p), (size_t)size); } -void sq_vm_free(void *p, SQUnsignedInteger size){ free(p); } +void sq_vm_free(void *p, SQUnsignedInteger size) { free(p); } +#endif diff --git a/src/ai/ai_instance.cpp b/src/ai/ai_instance.cpp index 41a8420853..edb7ebce0b 100644 --- a/src/ai/ai_instance.cpp +++ b/src/ai/ai_instance.cpp @@ -230,6 +230,7 @@ void AIInstance::Died() void AIInstance::LoadDummyScript() { + ScriptAllocatorScope alloc_scope(this->engine); extern void Script_CreateDummy(HSQUIRRELVM vm, StringID string, const char *type); Script_CreateDummy(this->engine->GetVM(), STR_ERROR_AI_NO_AI_FOUND, "AI"); } diff --git a/src/ai/ai_scanner.cpp b/src/ai/ai_scanner.cpp index 1d0fed1886..8bd3a43853 100644 --- a/src/ai/ai_scanner.cpp +++ b/src/ai/ai_scanner.cpp @@ -31,6 +31,8 @@ void AIScannerInfo::Initialize() { ScriptScanner::Initialize("AIScanner"); + ScriptAllocatorScope alloc_scope(this->engine); + /* Create the dummy AI */ free(this->main_script); this->main_script = stredup("%_dummy"); diff --git a/src/lang/english.txt b/src/lang/english.txt index cfe03b6d04..675617dcd4 100644 --- a/src/lang/english.txt +++ b/src/lang/english.txt @@ -1488,6 +1488,9 @@ STR_CONFIG_SETTING_AI_IN_MULTIPLAYER :Allow AIs in mu STR_CONFIG_SETTING_AI_IN_MULTIPLAYER_HELPTEXT :Allow AI computer players to participate in multiplayer games STR_CONFIG_SETTING_SCRIPT_MAX_OPCODES :#opcodes before scripts are suspended: {STRING2} STR_CONFIG_SETTING_SCRIPT_MAX_OPCODES_HELPTEXT :Maximum number of computation steps that a script can take in one turn +STR_CONFIG_SETTING_SCRIPT_MAX_MEMORY :Max memory usage per script: {STRING2} +STR_CONFIG_SETTING_SCRIPT_MAX_MEMORY_HELPTEXT :How much memory a single script may consume before it's forcibly terminated. This may need to be increased for large maps. +STR_CONFIG_SETTING_SCRIPT_MAX_MEMORY_VALUE :{COMMA} MiB STR_CONFIG_SETTING_SERVINT_ISPERCENT :Service intervals are in percents: {STRING2} STR_CONFIG_SETTING_SERVINT_ISPERCENT_HELPTEXT :Choose whether servicing of vehicles is triggered by the time passed since last service or by reliability dropping by a certain percentage of the maximum reliability diff --git a/src/saveload/saveload.h b/src/saveload/saveload.h index 7ac602a540..fd8b3695fc 100644 --- a/src/saveload/saveload.h +++ b/src/saveload/saveload.h @@ -300,6 +300,8 @@ enum SaveLoadVersion : uint16 { SLV_TREES_WATER_CLASS, ///< 213 PR#7405 WaterClass update for tree tiles. SLV_ROAD_TYPES, ///< 214 PR#6811 NewGRF road types. + SLV_SCRIPT_MEMLIMIT, ///< 215 PR#7516 Limit on AI/GS memory consumption. + SL_MAX_VERSION, ///< Highest possible saveload version }; diff --git a/src/script/api/script_object.cpp b/src/script/api/script_object.cpp index 67ab7e40dc..62a71574ff 100644 --- a/src/script/api/script_object.cpp +++ b/src/script/api/script_object.cpp @@ -38,7 +38,7 @@ static ScriptStorage *GetStorage() /* static */ ScriptInstance *ScriptObject::ActiveInstance::active = nullptr; -ScriptObject::ActiveInstance::ActiveInstance(ScriptInstance *instance) +ScriptObject::ActiveInstance::ActiveInstance(ScriptInstance *instance) : alc_scope(instance->engine) { this->last_active = ScriptObject::ActiveInstance::active; ScriptObject::ActiveInstance::active = instance; diff --git a/src/script/api/script_object.hpp b/src/script/api/script_object.hpp index 6725f0ed16..55dd5e39b7 100644 --- a/src/script/api/script_object.hpp +++ b/src/script/api/script_object.hpp @@ -18,6 +18,7 @@ #include "script_types.hpp" #include "../script_suspend.hpp" +#include "../squirrel.hpp" /** * The callback function for Mode-classes. @@ -48,6 +49,7 @@ protected: ~ActiveInstance(); private: ScriptInstance *last_active; ///< The active instance before we go instantiated. + ScriptAllocatorScope alc_scope; ///< Keep the correct allocator for the script instance activated static ScriptInstance *active; ///< The global current active instance. }; diff --git a/src/script/script_instance.cpp b/src/script/script_instance.cpp index 336d3366ae..9670e37c19 100644 --- a/src/script/script_instance.cpp +++ b/src/script/script_instance.cpp @@ -153,6 +153,8 @@ void ScriptInstance::Died() DEBUG(script, 0, "The script died unexpectedly."); this->is_dead = true; + this->last_allocated_memory = this->GetAllocatedMemory(); // Update cache + if (this->instance != nullptr) this->engine->ReleaseObject(this->instance); delete this->engine; this->instance = nullptr; @@ -698,3 +700,9 @@ void ScriptInstance::InsertEvent(class ScriptEvent *event) ScriptEventController::InsertEvent(event); } + +size_t ScriptInstance::GetAllocatedMemory() const +{ + if (this->engine == nullptr) return this->last_allocated_memory; + return this->engine->GetAllocatedMemory(); +} diff --git a/src/script/script_instance.hpp b/src/script/script_instance.hpp index 56e830367c..653ec38840 100644 --- a/src/script/script_instance.hpp +++ b/src/script/script_instance.hpp @@ -198,6 +198,8 @@ public: */ bool IsSleeping() { return this->suspend != 0; } + size_t GetAllocatedMemory() const; + protected: class Squirrel *engine; ///< A wrapper around the squirrel vm. const char *versionAPI; ///< Current API used by this script. @@ -241,6 +243,7 @@ private: int suspend; ///< The amount of ticks to suspend this script before it's allowed to continue. bool is_paused; ///< Is the script paused? (a paused script will not be executed until unpaused) Script_SuspendCallbackProc *callback; ///< Callback that should be called in the next tick the script runs. + size_t last_allocated_memory; ///< Last known allocated memory value (for display for crashed scripts) /** * Call the script Load function if it exists and data was loaded diff --git a/src/script/squirrel.cpp b/src/script/squirrel.cpp index fd0478bb52..ea63944ea9 100644 --- a/src/script/squirrel.cpp +++ b/src/script/squirrel.cpp @@ -10,17 +10,127 @@ /** @file squirrel.cpp the implementation of the Squirrel class. It handles all Squirrel-stuff and gives a nice API back to work with. */ #include +#include #include "../stdafx.h" #include "../debug.h" #include "squirrel_std.hpp" #include "../fileio_func.h" #include "../string_func.h" +#include "script_fatalerror.hpp" +#include "../settings_type.h" #include #include <../squirrel/sqpcheader.h> #include <../squirrel/sqvm.h> +#include "../core/alloc_func.hpp" #include "../safeguards.h" +/* + * If changing the call paths into the scripting engine, define this symbol to enable full debugging of allocations. + * This lets you track whether the allocator context is being switched correctly in all call paths. +#define SCRIPT_DEBUG_ALLOCATIONS +*/ + +struct ScriptAllocator { + size_t allocated_size; ///< Sum of allocated data size + size_t allocation_limit; ///< Maximum this allocator may use before allocations fail + + static const size_t SAFE_LIMIT = 0x8000000; ///< 128 MiB, a safe choice for almost any situation + +#ifdef SCRIPT_DEBUG_ALLOCATIONS + std::map allocations; +#endif + + void CheckLimit() const + { + if (this->allocated_size > this->allocation_limit) throw Script_FatalError("Maximum memory allocation exceeded"); + } + + void *Malloc(SQUnsignedInteger size) + { + void *p = MallocT(size); + this->allocated_size += size; + +#ifdef SCRIPT_DEBUG_ALLOCATIONS + assert(p != nullptr); + assert(this->allocations.find(p) == this->allocations.end()); + this->allocations[p] = size; +#endif + + return p; + } + + void *Realloc(void *p, SQUnsignedInteger oldsize, SQUnsignedInteger size) + { + if (p == nullptr) { + return this->Malloc(size); + } + if (size == 0) { + this->Free(p, oldsize); + return nullptr; + } + +#ifdef SCRIPT_DEBUG_ALLOCATIONS + assert(this->allocations[p] == oldsize); + this->allocations.erase(p); +#endif + + void *new_p = ReallocT(static_cast(p), size); + + this->allocated_size -= oldsize; + this->allocated_size += size; + +#ifdef SCRIPT_DEBUG_ALLOCATIONS + assert(new_p != nullptr); + assert(this->allocations.find(p) == this->allocations.end()); + this->allocations[new_p] = size; +#endif + + return new_p; + } + + void Free(void *p, SQUnsignedInteger size) + { + if (p == nullptr) return; + free(p); + +#ifdef SCRIPT_DEBUG_ALLOCATIONS + assert(this->allocations.at(p) == size); + this->allocations.erase(p); +#endif + } + + ScriptAllocator() + { + this->allocated_size = 0; + this->allocation_limit = static_cast(_settings_game.script.script_max_memory_megabytes) << 20; + if (this->allocation_limit == 0) this->allocation_limit = SAFE_LIMIT; // in case the setting is somehow zero + } + + ~ScriptAllocator() + { +#ifdef SCRIPT_DEBUG_ALLOCATIONS + assert(this->allocations.size() == 0); +#endif + } +}; + +ScriptAllocator *_squirrel_allocator = nullptr; + +/* See 3rdparty/squirrel/squirrel/sqmem.cpp for the default allocator implementation, which this overrides */ +#ifndef SQUIRREL_DEFAULT_ALLOCATOR +void *sq_vm_malloc(SQUnsignedInteger size) { return _squirrel_allocator->Malloc(size); } +void *sq_vm_realloc(void *p, SQUnsignedInteger oldsize, SQUnsignedInteger size) { return _squirrel_allocator->Realloc(p, oldsize, size); } +void sq_vm_free(void *p, SQUnsignedInteger size) { _squirrel_allocator->Free(p, size); } +#endif + +size_t Squirrel::GetAllocatedMemory() const noexcept +{ + assert(this->allocator != nullptr); + return this->allocator->allocated_size; +} + + void Squirrel::CompileError(HSQUIRRELVM vm, const SQChar *desc, const SQChar *source, SQInteger line, SQInteger column) { SQChar buf[1024]; @@ -115,6 +225,8 @@ void Squirrel::PrintFunc(HSQUIRRELVM vm, const SQChar *s, ...) void Squirrel::AddMethod(const char *method_name, SQFUNCTION proc, uint nparam, const char *params, void *userdata, int size) { + ScriptAllocatorScope alloc_scope(this); + sq_pushstring(this->vm, method_name, -1); if (size != 0) { @@ -130,6 +242,8 @@ void Squirrel::AddMethod(const char *method_name, SQFUNCTION proc, uint nparam, void Squirrel::AddConst(const char *var_name, int value) { + ScriptAllocatorScope alloc_scope(this); + sq_pushstring(this->vm, var_name, -1); sq_pushinteger(this->vm, value); sq_newslot(this->vm, -3, SQTrue); @@ -137,6 +251,8 @@ void Squirrel::AddConst(const char *var_name, int value) void Squirrel::AddConst(const char *var_name, bool value) { + ScriptAllocatorScope alloc_scope(this); + sq_pushstring(this->vm, var_name, -1); sq_pushbool(this->vm, value); sq_newslot(this->vm, -3, SQTrue); @@ -144,6 +260,8 @@ void Squirrel::AddConst(const char *var_name, bool value) void Squirrel::AddClassBegin(const char *class_name) { + ScriptAllocatorScope alloc_scope(this); + sq_pushroottable(this->vm); sq_pushstring(this->vm, class_name, -1); sq_newclass(this->vm, SQFalse); @@ -151,6 +269,8 @@ void Squirrel::AddClassBegin(const char *class_name) void Squirrel::AddClassBegin(const char *class_name, const char *parent_class) { + ScriptAllocatorScope alloc_scope(this); + sq_pushroottable(this->vm); sq_pushstring(this->vm, class_name, -1); sq_pushstring(this->vm, parent_class, -1); @@ -164,6 +284,8 @@ void Squirrel::AddClassBegin(const char *class_name, const char *parent_class) void Squirrel::AddClassEnd() { + ScriptAllocatorScope alloc_scope(this); + sq_newslot(vm, -3, SQFalse); sq_pop(vm, 1); } @@ -171,6 +293,8 @@ void Squirrel::AddClassEnd() bool Squirrel::MethodExists(HSQOBJECT instance, const char *method_name) { assert(!this->crashed); + ScriptAllocatorScope alloc_scope(this); + int top = sq_gettop(this->vm); /* Go to the instance-root */ sq_pushobject(this->vm, instance); @@ -187,6 +311,8 @@ bool Squirrel::MethodExists(HSQOBJECT instance, const char *method_name) bool Squirrel::Resume(int suspend) { assert(!this->crashed); + ScriptAllocatorScope alloc_scope(this); + /* Did we use more operations than we should have in the * previous tick? If so, subtract that from the current run. */ if (this->overdrawn_ops > 0 && suspend > 0) { @@ -200,23 +326,29 @@ bool Squirrel::Resume(int suspend) this->crashed = !sq_resumecatch(this->vm, suspend); this->overdrawn_ops = -this->vm->_ops_till_suspend; + this->allocator->CheckLimit(); return this->vm->_suspended != 0; } void Squirrel::ResumeError() { assert(!this->crashed); + ScriptAllocatorScope alloc_scope(this); sq_resumeerror(this->vm); } void Squirrel::CollectGarbage() { + ScriptAllocatorScope alloc_scope(this); sq_collectgarbage(this->vm); } bool Squirrel::CallMethod(HSQOBJECT instance, const char *method_name, HSQOBJECT *ret, int suspend) { assert(!this->crashed); + ScriptAllocatorScope alloc_scope(this); + this->allocator->CheckLimit(); + /* Store the stack-location for the return value. We need to * restore this after saving or the stack will be corrupted * if we're in the middle of a DoCommand. */ @@ -325,17 +457,20 @@ bool Squirrel::CallBoolMethod(HSQOBJECT instance, const char *method_name, bool bool Squirrel::CreateClassInstance(const char *class_name, void *real_instance, HSQOBJECT *instance) { + ScriptAllocatorScope alloc_scope(this); return Squirrel::CreateClassInstanceVM(this->vm, class_name, real_instance, instance, nullptr); } Squirrel::Squirrel(const char *APIName) : - APIName(APIName) + APIName(APIName), allocator(new ScriptAllocator()) { this->Initialize(); } void Squirrel::Initialize() { + ScriptAllocatorScope alloc_scope(this); + this->global_pointer = nullptr; this->print_func = nullptr; this->crashed = false; @@ -432,6 +567,8 @@ static SQInteger _io_file_read(SQUserPointer file, SQUserPointer buf, SQInteger SQRESULT Squirrel::LoadFile(HSQUIRRELVM vm, const char *filename, SQBool printerror) { + ScriptAllocatorScope alloc_scope(this); + FILE *file; size_t size; if (strncmp(this->GetAPIName(), "AI", 2) == 0) { @@ -517,6 +654,8 @@ SQRESULT Squirrel::LoadFile(HSQUIRRELVM vm, const char *filename, SQBool printer bool Squirrel::LoadScript(HSQUIRRELVM vm, const char *script, bool in_root) { + ScriptAllocatorScope alloc_scope(this); + /* Make sure we are always in the root-table */ if (in_root) sq_pushroottable(vm); @@ -549,6 +688,8 @@ Squirrel::~Squirrel() void Squirrel::Uninitialize() { + ScriptAllocatorScope alloc_scope(this); + /* Clean up the stuff */ sq_pop(this->vm, 1); sq_close(this->vm); @@ -562,6 +703,8 @@ void Squirrel::Reset() void Squirrel::InsertResult(bool result) { + ScriptAllocatorScope alloc_scope(this); + sq_pushbool(this->vm, result); if (this->IsSuspended()) { // Called before resuming a suspended script? vm->GetAt(vm->_stackbase + vm->_suspended_target) = vm->GetUp(-1); @@ -571,6 +714,8 @@ void Squirrel::InsertResult(bool result) void Squirrel::InsertResult(int result) { + ScriptAllocatorScope alloc_scope(this); + sq_pushinteger(this->vm, result); if (this->IsSuspended()) { // Called before resuming a suspended script? vm->GetAt(vm->_stackbase + vm->_suspended_target) = vm->GetUp(-1); @@ -600,6 +745,7 @@ void Squirrel::CrashOccurred() bool Squirrel::CanSuspend() { + ScriptAllocatorScope alloc_scope(this); return sq_can_suspend(this->vm); } diff --git a/src/script/squirrel.hpp b/src/script/squirrel.hpp index bec41f575d..d0d02e80f8 100644 --- a/src/script/squirrel.hpp +++ b/src/script/squirrel.hpp @@ -20,7 +20,11 @@ enum ScriptType { ST_GS, ///< The script is for Game scripts. }; +struct ScriptAllocator; + class Squirrel { + friend class ScriptAllocatorScope; + private: typedef void (SQPrintFunc)(bool error_msg, const SQChar *message); @@ -30,6 +34,7 @@ private: bool crashed; ///< True if the squirrel script made an error. int overdrawn_ops; ///< The amount of operations we have overdrawn. const char *APIName; ///< Name of the API used for this squirrel. + std::unique_ptr allocator; ///< Allocator object used by this script. /** * The internal RunError handler. It looks up the real error and calls RunError with it. @@ -272,6 +277,31 @@ public: * Completely reset the engine; start from scratch. */ void Reset(); + + /** + * Get number of bytes allocated by this VM. + */ + size_t GetAllocatedMemory() const noexcept; +}; + + +extern ScriptAllocator *_squirrel_allocator; + +class ScriptAllocatorScope { + ScriptAllocator *old_allocator; + +public: + ScriptAllocatorScope(const Squirrel *engine) + { + this->old_allocator = _squirrel_allocator; + /* This may get called with a nullptr engine, in case of a crashed script */ + _squirrel_allocator = engine != nullptr ? engine->allocator.get() : nullptr; + } + + ~ScriptAllocatorScope() + { + _squirrel_allocator = this->old_allocator; + } }; #endif /* SQUIRREL_HPP */ diff --git a/src/settings_gui.cpp b/src/settings_gui.cpp index a6993cbf6a..bde6e05579 100644 --- a/src/settings_gui.cpp +++ b/src/settings_gui.cpp @@ -1757,6 +1757,7 @@ static SettingsContainer &GetSettingsTree() { npc->Add(new SettingEntry("script.settings_profile")); npc->Add(new SettingEntry("script.script_max_opcode_till_suspend")); + npc->Add(new SettingEntry("script.script_max_memory_megabytes")); npc->Add(new SettingEntry("difficulty.competitor_speed")); npc->Add(new SettingEntry("ai.ai_in_multiplayer")); npc->Add(new SettingEntry("ai.ai_disable_veh_train")); diff --git a/src/settings_type.h b/src/settings_type.h index f2fbd283cd..9315cdd968 100644 --- a/src/settings_type.h +++ b/src/settings_type.h @@ -339,6 +339,7 @@ struct AISettings { struct ScriptSettings { uint8 settings_profile; ///< difficulty profile to set initial settings of scripts, esp. random AIs uint32 script_max_opcode_till_suspend; ///< max opcode calls till scripts will suspend + uint32 script_max_memory_megabytes; ///< limit on memory a single script instance may have allocated }; /** Settings related to the new pathfinder. */ diff --git a/src/table/settings.ini b/src/table/settings.ini index 9e3fa00547..800d6f79c1 100644 --- a/src/table/settings.ini +++ b/src/table/settings.ini @@ -1555,6 +1555,21 @@ strhelp = STR_CONFIG_SETTING_SCRIPT_MAX_OPCODES_HELPTEXT strval = STR_JUST_COMMA cat = SC_EXPERT +[SDT_VAR] +base = GameSettings +var = script.script_max_memory_megabytes +type = SLE_UINT32 +from = SLV_SCRIPT_MEMLIMIT +guiflags = SGF_NEWGAME_ONLY +def = 1024 +min = 8 +max = 8192 +interval = 8 +str = STR_CONFIG_SETTING_SCRIPT_MAX_MEMORY +strhelp = STR_CONFIG_SETTING_SCRIPT_MAX_MEMORY_HELPTEXT +strval = STR_CONFIG_SETTING_SCRIPT_MAX_MEMORY_VALUE +cat = SC_EXPERT + ## [SDT_VAR] base = GameSettings