diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs index 17546cbd21..ef5f88aa01 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs @@ -2223,7 +2223,7 @@ namespace Microsoft.ML.OnnxRuntime.Tests } [Fact] - private void TestWeightSharingBetweenSessions() + private void TestSharingOfInitializerAndItsPrepackedVersion() { string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "matmul_1.onnx"); diff --git a/include/onnxruntime/core/framework/allocator.h b/include/onnxruntime/core/framework/allocator.h index 99a345512b..3820d79606 100644 --- a/include/onnxruntime/core/framework/allocator.h +++ b/include/onnxruntime/core/framework/allocator.h @@ -16,20 +16,20 @@ struct OrtArenaCfg { arena_extend_strategy(-1), initial_chunk_size_bytes(-1), max_dead_bytes_per_chunk(-1), - initial_regrowth_chunk_size_bytes(-1) {} + initial_growth_chunk_size_bytes(-1) {} OrtArenaCfg(size_t max_mem, int arena_extend_strategy, int initial_chunk_size_bytes, - int max_dead_bytes_per_chunk, int initial_regrowth_chunk_size_bytes) + int max_dead_bytes_per_chunk, int initial_growth_chunk_size_bytes) : max_mem(max_mem), arena_extend_strategy(arena_extend_strategy), initial_chunk_size_bytes(initial_chunk_size_bytes), max_dead_bytes_per_chunk(max_dead_bytes_per_chunk), - initial_regrowth_chunk_size_bytes(initial_regrowth_chunk_size_bytes) {} + initial_growth_chunk_size_bytes(initial_growth_chunk_size_bytes) {} - size_t max_mem; // use 0 to allow ORT to choose the default - int arena_extend_strategy; // use -1 to allow ORT to choose the default, 0 = kNextPowerOfTwo, 1 = kSameAsRequested - int initial_chunk_size_bytes; // use -1 to allow ORT to choose the default - int max_dead_bytes_per_chunk; // use -1 to allow ORT to choose the default - int initial_regrowth_chunk_size_bytes; // use -1 to allow ORT to choose the default + size_t max_mem; // use 0 to allow ORT to choose the default + int arena_extend_strategy; // use -1 to allow ORT to choose the default, 0 = kNextPowerOfTwo, 1 = kSameAsRequested + int initial_chunk_size_bytes; // use -1 to allow ORT to choose the default + int max_dead_bytes_per_chunk; // use -1 to allow ORT to choose the default + int initial_growth_chunk_size_bytes; // use -1 to allow ORT to choose the default }; namespace onnxruntime { diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index b57ed2f570..714836a752 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -1310,7 +1310,7 @@ struct OrtApi { Ultimately, the first allocation size is determined by the allocation memory request. * "max_dead_bytes_per_chunk": Threshold of unused memory in an allocated chunk of arena memory after crossing which the current chunk is chunked into 2. - * "initial_regrowth_chunk_size_bytes": (Possible) Size of the second allocation in the arena. + * "initial_growth_chunk_size_bytes": (Possible) Size of the second allocation in the arena. Only relevant if arena strategy is `kNextPowerOfTwo`. Use -1 to allow ORT to choose the default. Ultimately, the allocation size is determined by the allocation memory request. Further allocation sizes are governed by the arena extend strategy. diff --git a/onnxruntime/core/framework/allocatormgr.cc b/onnxruntime/core/framework/allocatormgr.cc index 630c6dd635..9157130bcc 100644 --- a/onnxruntime/core/framework/allocatormgr.cc +++ b/onnxruntime/core/framework/allocatormgr.cc @@ -31,9 +31,9 @@ AllocatorPtr CreateAllocator(const AllocatorCreationInfo& info) { int max_dead_bytes_per_chunk = info.arena_cfg.max_dead_bytes_per_chunk == -1 ? BFCArena::DEFAULT_MAX_DEAD_BYTES_PER_CHUNK : info.arena_cfg.max_dead_bytes_per_chunk; - int initial_regrowth_chunk_size_bytes = info.arena_cfg.initial_regrowth_chunk_size_bytes == -1 - ? BFCArena::DEFAULT_INITIAL_REGROWTH_CHUNK_SIZE_BYTES - : info.arena_cfg.initial_regrowth_chunk_size_bytes; + int initial_growth_chunk_size_bytes = info.arena_cfg.initial_growth_chunk_size_bytes == -1 + ? BFCArena::DEFAULT_INITIAL_GROWTH_CHUNK_SIZE_BYTES + : info.arena_cfg.initial_growth_chunk_size_bytes; ArenaExtendStrategy arena_extend_str; switch (info.arena_cfg.arena_extend_strategy) { case static_cast(ArenaExtendStrategy::kSameAsRequested): @@ -58,7 +58,7 @@ AllocatorPtr CreateAllocator(const AllocatorCreationInfo& info) { arena_extend_str, initial_chunk_size_bytes, max_dead_bytes_per_chunk, - initial_regrowth_chunk_size_bytes)); + initial_growth_chunk_size_bytes)); #endif } diff --git a/onnxruntime/core/framework/bfc_arena.cc b/onnxruntime/core/framework/bfc_arena.cc index c8683e1069..eab62c08b0 100644 --- a/onnxruntime/core/framework/bfc_arena.cc +++ b/onnxruntime/core/framework/bfc_arena.cc @@ -10,7 +10,7 @@ BFCArena::BFCArena(std::unique_ptr resource_allocator, ArenaExtendStrategy arena_extend_strategy, int initial_chunk_size_bytes, int max_dead_bytes_per_chunk, - int initial_regrowth_chunk_size_bytes) + int initial_growth_chunk_size_bytes) : IArenaAllocator(OrtMemoryInfo(resource_allocator->Info().name, OrtAllocatorType::OrtArenaAllocator, resource_allocator->Info().device, @@ -21,11 +21,11 @@ BFCArena::BFCArena(std::unique_ptr resource_allocator, next_allocation_id_(1), initial_chunk_size_bytes_(initial_chunk_size_bytes), max_dead_bytes_per_chunk_(max_dead_bytes_per_chunk), - initial_regrowth_chunk_size_bytes_(initial_regrowth_chunk_size_bytes) { + initial_growth_chunk_size_bytes_(initial_growth_chunk_size_bytes) { LOGS_DEFAULT(INFO) << "Creating BFCArena for " << device_allocator_->Info().name << " with following configs: initial_chunk_size_bytes: " << initial_chunk_size_bytes_ << " max_dead_bytes_per_chunk: " << max_dead_bytes_per_chunk_ - << " initial_regrowth_chunk_size_bytes: " << initial_regrowth_chunk_size_bytes_ + << " initial_growth_chunk_size_bytes: " << initial_growth_chunk_size_bytes_ << " memory limit: " << total_memory << " arena_extend_strategy: " << static_cast(arena_extend_strategy); @@ -491,7 +491,7 @@ Status BFCArena::Shrink() { // Will affect how the arena grows if the arena extend strategy is kNextPowerOfTwo // In case the extend strategy is kSameAsRequested, the arena growth is exactly the size of the memory request itself - curr_region_allocation_bytes_ = initial_regrowth_chunk_size_bytes_; + curr_region_allocation_bytes_ = initial_growth_chunk_size_bytes_; return Status::OK(); } diff --git a/onnxruntime/core/framework/bfc_arena.h b/onnxruntime/core/framework/bfc_arena.h index 534a469eb1..914af6899b 100644 --- a/onnxruntime/core/framework/bfc_arena.h +++ b/onnxruntime/core/framework/bfc_arena.h @@ -55,7 +55,7 @@ class BFCArena : public IArenaAllocator { static const ArenaExtendStrategy DEFAULT_ARENA_EXTEND_STRATEGY = ArenaExtendStrategy::kNextPowerOfTwo; static const int DEFAULT_INITIAL_CHUNK_SIZE_BYTES = 1 * 1024 * 1024; static const int DEFAULT_MAX_DEAD_BYTES_PER_CHUNK = 128 * 1024 * 1024; - static const int DEFAULT_INITIAL_REGROWTH_CHUNK_SIZE_BYTES = 2 * 1024 * 1024; + static const int DEFAULT_INITIAL_GROWTH_CHUNK_SIZE_BYTES = 2 * 1024 * 1024; static const size_t DEFAULT_MAX_MEM = std::numeric_limits::max(); BFCArena(std::unique_ptr resource_allocator, @@ -63,7 +63,7 @@ class BFCArena : public IArenaAllocator { ArenaExtendStrategy arena_extend_strategy = DEFAULT_ARENA_EXTEND_STRATEGY, int initial_chunk_size_bytes = DEFAULT_INITIAL_CHUNK_SIZE_BYTES, int max_dead_bytes_per_chunk = DEFAULT_MAX_DEAD_BYTES_PER_CHUNK, - int initial_regrowth_chunk_size_bytes = DEFAULT_INITIAL_REGROWTH_CHUNK_SIZE_BYTES); + int initial_growth_chunk_size_bytes = DEFAULT_INITIAL_GROWTH_CHUNK_SIZE_BYTES); ~BFCArena() override; @@ -78,7 +78,7 @@ class BFCArena : public IArenaAllocator { // Frees all allocation regions in which no chunk is in use. // Does not free any reserved chunks. // Resets the size that the arena will grow by in the next allocation to - // `initial_regrowth_chunk_size_bytes_` but ultimately all + // `initial_growth_chunk_size_bytes_` but ultimately all // future allocation sizes are determined by the arena growth strategy // and the allocation request. Status Shrink() override; @@ -475,7 +475,7 @@ class BFCArena : public IArenaAllocator { const int initial_chunk_size_bytes_; const int max_dead_bytes_per_chunk_; - const int initial_regrowth_chunk_size_bytes_; + const int initial_growth_chunk_size_bytes_; // This flag is only relevant if Shrink() is invoked. // This is a boolean flag that controls whether the first allocation region diff --git a/onnxruntime/core/session/environment.cc b/onnxruntime/core/session/environment.cc index 0c3f38fbf4..3bd28d630a 100644 --- a/onnxruntime/core/session/environment.cc +++ b/onnxruntime/core/session/environment.cc @@ -94,7 +94,7 @@ Status Environment::CreateAndRegisterAllocator(const OrtMemoryInfo& mem_info, co int arena_extend_strategy = -1; int initial_chunk_size_bytes = -1; int max_dead_bytes_per_chunk = -1; - int initial_regrowth_chunk_size_bytes = -1; + int initial_growth_chunk_size_bytes = -1; // override with values from the user supplied arena_cfg object if (arena_cfg) { @@ -110,11 +110,11 @@ Status Environment::CreateAndRegisterAllocator(const OrtMemoryInfo& mem_info, co initial_chunk_size_bytes = arena_cfg->initial_chunk_size_bytes; max_dead_bytes_per_chunk = arena_cfg->max_dead_bytes_per_chunk; - initial_regrowth_chunk_size_bytes = arena_cfg->initial_regrowth_chunk_size_bytes; + initial_growth_chunk_size_bytes = arena_cfg->initial_growth_chunk_size_bytes; } OrtArenaCfg l_arena_cfg{max_mem, arena_extend_strategy, initial_chunk_size_bytes, max_dead_bytes_per_chunk, - initial_regrowth_chunk_size_bytes}; + initial_growth_chunk_size_bytes}; AllocatorCreationInfo alloc_creation_info{ [mem_info](int) { return std::make_unique(mem_info); }, 0, diff --git a/onnxruntime/core/session/onnxruntime_c_api.cc b/onnxruntime/core/session/onnxruntime_c_api.cc index 42a9612b7b..be818df5ef 100644 --- a/onnxruntime/core/session/onnxruntime_c_api.cc +++ b/onnxruntime/core/session/onnxruntime_c_api.cc @@ -1889,8 +1889,8 @@ ORT_API_STATUS_IMPL(OrtApis::CreateArenaCfgV2, _In_reads_(num_keys) const char* cfg->initial_chunk_size_bytes = static_cast(arena_config_values[i]); } else if (strcmp(arena_config_keys[i], "max_dead_bytes_per_chunk") == 0) { cfg->max_dead_bytes_per_chunk = static_cast(arena_config_values[i]); - } else if (strcmp(arena_config_keys[i], "initial_regrowth_chunk_size_bytes") == 0) { - cfg->initial_regrowth_chunk_size_bytes = static_cast(arena_config_values[i]); + } else if (strcmp(arena_config_keys[i], "initial_growth_chunk_size_bytes") == 0) { + cfg->initial_growth_chunk_size_bytes = static_cast(arena_config_values[i]); } else { std::ostringstream oss; oss << "Invalid key found: " << arena_config_keys[i]; diff --git a/onnxruntime/test/shared_lib/test_inference.cc b/onnxruntime/test/shared_lib/test_inference.cc index ad116f75b0..4f04588d93 100644 --- a/onnxruntime/test/shared_lib/test_inference.cc +++ b/onnxruntime/test/shared_lib/test_inference.cc @@ -1327,7 +1327,7 @@ TEST(CApiTest, TestSharedAllocatorUsingCreateAndRegisterAllocator) { nullptr); } -TEST(CApiTest, TestSharingOfInitializerWithPrepackedWeightsCaching) { +TEST(CApiTest, TestSharingOfInitializerAndItsPrepackedVersion) { // simple inference test // prepare inputs std::vector inputs(1); @@ -1442,7 +1442,7 @@ TEST(CApiTest, TestIncorrectInputTypeToModel_SequenceTensors) { } #endif -TEST(CApiTest, allocate_initializers_from_non_arena_memory) { +TEST(CApiTest, AllocateInitializersFromNonArenaMemory) { Ort::SessionOptions session_options; #ifdef USE_CUDA @@ -1464,12 +1464,12 @@ TEST(CApiTest, allocate_initializers_from_non_arena_memory) { #ifdef USE_CUDA // Usage example showing how to use CreateArenaCfgV2() API to configure the default memory CUDA arena allocator -TEST(CApiTest, configure_cuda_arena_and_demonstrate_memory_arena_shrinkage) { +TEST(CApiTest, ConfigureCudaArenaAndDemonstrateMemoryArenaShrinkage) { const auto& api = Ort::GetApi(); Ort::SessionOptions session_options; - const char* keys[] = {"max_mem", "arena_extend_strategy", "initial_chunk_size_bytes", "max_dead_bytes_per_chunk", "initial_regrowth_chunk_size_bytes"}; + const char* keys[] = {"max_mem", "arena_extend_strategy", "initial_chunk_size_bytes", "max_dead_bytes_per_chunk", "initial_growth_chunk_size_bytes"}; const size_t values[] = {0 /*let ort pick default max memory*/, 0, 1024, 0, 256}; OrtArenaCfg* arena_cfg = nullptr;