saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-29 20:14:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Delete path.h (#21211)

Browse source 
### Description
Delete path.h and replace all occurrences of onnxruntime::Path with
std::filesystem::path.
Previously we couldn't use C++17's std::filesystem because it was not
supported in iOS 12(which was released in 2018). Now we dropped the
support for iOS 12.

### Motivation and Context
To simplify code. For example, if an EP wants to use the Path class, now
it can directly use it without going through a wrapper. And the standard
implementation can handle various path types better. (We didn't take
much consideration on UNC path, "/" as a path separator on Windows,
etc).
This commit is contained in:
Changming Sun 2024-07-04 00:54:13 -07:00 committed by GitHub
parent 40d4b2ec75
commit 07c429191e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
22 changed files with 25 additions and 734 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
include/onnxruntime/core/graph/graph.h
View file

@ -17,12 +17,12 @@
#include "core/common/gsl.h"
#include "core/common/common.h"
#include "core/common/path_string.h"
#include "core/common/const_pointer_container.h"
#if !defined(ORT_MINIMAL_BUILD)
#include "core/common/inlined_containers.h"
#endif
#include "core/common/inlined_containers_fwd.h"
#include "core/common/path.h"
#include "core/common/span_utils.h"
#include "core/common/status.h"
#include "core/common/logging/logging.h"

308
onnxruntime/core/common/path.cc
View file

@ -1,308 +0,0 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "core/common/path.h"
#include <algorithm>
#include <array>
namespace onnxruntime {
namespace {
constexpr auto k_dot = ORT_TSTR(".");
constexpr auto k_dotdot = ORT_TSTR("..");
constexpr std::array<PathChar, 2> k_valid_path_separators{
ORT_TSTR('/'), ORT_TSTR('\\')};
constexpr bool IsPreferredPathSeparator(PathChar c) {
return c == k_preferred_path_separator;
}
PathString NormalizePathSeparators(const PathString& path) {
PathString result{};
std::replace_copy_if(
path.begin(), path.end(), std::back_inserter(result),
[](PathChar c) {
return std::find(
k_valid_path_separators.begin(),
k_valid_path_separators.end(),
c) != k_valid_path_separators.end();
},
k_preferred_path_separator);
return result;
}
// parse component and trailing path separator
PathString::const_iterator ParsePathComponent(
PathString::const_iterator begin, PathString::const_iterator end,
PathString::const_iterator& component_end, bool* has_trailing_separator) {
component_end = std::find_if(begin, end, IsPreferredPathSeparator);
const auto sep_end = std::find_if_not(component_end, end, IsPreferredPathSeparator);
if (has_trailing_separator) *has_trailing_separator = sep_end != component_end;
return sep_end;
}
#ifdef _WIN32
Status ParsePathRoot(
const PathString& path,
PathString& root, bool& has_root_dir, size_t& num_parsed_chars) {
// assume NormalizePathSeparators() has been called
// drive letter
if (path.size() > 1 &&
(ORT_TSTR('a') <= path[0] && path[0] <= ORT_TSTR('z') ||
ORT_TSTR('A') <= path[0] && path[0] <= ORT_TSTR('Z')) &&
path[1] == ORT_TSTR(':')) {
const auto root_dir_begin = path.begin() + 2;
const auto root_dir_end = std::find_if_not(root_dir_begin, path.end(), IsPreferredPathSeparator);
root = path.substr(0, 2);
has_root_dir = root_dir_begin != root_dir_end;
num_parsed_chars = std::distance(path.begin(), root_dir_end);
return Status::OK();
}
// leading path separator
auto curr_it = std::find_if_not(path.begin(), path.end(), IsPreferredPathSeparator);
const auto num_initial_seps = std::distance(path.begin(), curr_it);
if (num_initial_seps == 2) {
// "\\server_name\share_name\"
// after "\\", parse 2 path components with trailing separators
PathString::const_iterator component_end;
bool has_trailing_separator;
curr_it = ParsePathComponent(curr_it, path.end(), component_end, &has_trailing_separator);
ORT_RETURN_IF_NOT(has_trailing_separator, "Failed to parse path root: ", ToUTF8String(path));
curr_it = ParsePathComponent(curr_it, path.end(), component_end, &has_trailing_separator);
ORT_RETURN_IF_NOT(has_trailing_separator, "Failed to parse path root: ", ToUTF8String(path));
root.assign(path.begin(), component_end);
has_root_dir = true;
num_parsed_chars = std::distance(path.begin(), curr_it);
} else {
// "\", ""
root.clear();
has_root_dir = num_initial_seps > 0;
num_parsed_chars = num_initial_seps;
}
return Status::OK();
}
#else // POSIX
Status ParsePathRoot(
const PathString& path,
PathString& root, bool& has_root_dir, size_t& num_parsed_chars) {
// assume NormalizePathSeparators() has been called
auto curr_it = std::find_if_not(path.begin(), path.end(), IsPreferredPathSeparator);
const auto num_initial_seps = std::distance(path.begin(), curr_it);
if (num_initial_seps == 2) {
// "//root_name/"
// after "//", parse path component with trailing separator
PathString::const_iterator component_end;
bool has_trailing_separator;
curr_it = ParsePathComponent(curr_it, path.end(), component_end, &has_trailing_separator);
ORT_RETURN_IF_NOT(has_trailing_separator, "Failed to parse path root: ", ToUTF8String(path));
root.assign(path.begin(), component_end);
has_root_dir = true;
num_parsed_chars = std::distance(path.begin(), curr_it);
} else {
// "/", ""
root.clear();
has_root_dir = num_initial_seps > 0;
num_parsed_chars = num_initial_seps;
}
return Status::OK();
}
#endif
} // namespace
Status Path::Parse(const PathString& original_path_str, Path& path) {
Path result{};
// normalize separators
const PathString path_str = NormalizePathSeparators(original_path_str);
// parse root
size_t root_length = 0;
ORT_RETURN_IF_ERROR(ParsePathRoot(
path_str, result.root_name_, result.has_root_dir_, root_length));
// parse components
PathString::const_iterator component_begin = path_str.begin() + root_length;
while (component_begin != path_str.end()) {
PathString::const_iterator component_end;
PathString::const_iterator next_component_begin = ParsePathComponent(
component_begin, path_str.end(), component_end, nullptr);
result.components_.emplace_back(component_begin, component_end);
component_begin = next_component_begin;
}
path = std::move(result);
return Status::OK();
}
Path Path::Parse(const PathString& path_str) {
Path path{};
const auto status = Parse(path_str, path);
ORT_ENFORCE(status.IsOK(), status.ErrorMessage());
return path;
}
PathString Path::ToPathString() const {
PathString result = GetRootPathString();
const size_t components_size = components_.size();
for (size_t i = 0; i < components_size; ++i) {
result += components_[i];
if (i + 1 < components_size) result += k_preferred_path_separator;
}
return result;
}
PathString Path::GetRootPathString() const {
return has_root_dir_ ? root_name_ + k_preferred_path_separator : root_name_;
}
bool Path::IsEmpty() const {
return !has_root_dir_ && root_name_.empty() && components_.empty();
}
bool Path::IsAbsolute() const {
#ifdef _WIN32
return has_root_dir_ && !root_name_.empty();
#else // POSIX
return has_root_dir_;
#endif
}
Path Path::ParentPath() const {
Path parent{*this};
if (!parent.components_.empty()) parent.components_.pop_back();
return parent;
}
Path& Path::Normalize() {
if (IsEmpty()) return *this;
// handle . and ..
std::vector<PathString> normalized_components{};
for (const auto& component : components_) {
// ignore .
if (component == k_dot) continue;
// handle .. which backtracks over previous component
if (component == k_dotdot) {
if (!normalized_components.empty() &&
normalized_components.back() != k_dotdot) {
normalized_components.pop_back();
continue;
}
}
normalized_components.emplace_back(component);
}
// remove leading ..'s if root dir present
if (has_root_dir_) {
const auto first_non_dotdot_it = std::find_if(
normalized_components.begin(), normalized_components.end(),
[](const PathString& component) { return component != k_dotdot; });
normalized_components.erase(
normalized_components.begin(), first_non_dotdot_it);
}
// if empty at this point, add a dot
if (!has_root_dir_ && root_name_.empty() && normalized_components.empty()) {
normalized_components.emplace_back(k_dot);
}
components_ = std::move(normalized_components);
return *this;
}
Path& Path::Append(const Path& other) {
if (other.IsAbsolute() ||
(!other.root_name_.empty() && other.root_name_ != root_name_)) {
return *this = other;
}
if (other.has_root_dir_) {
has_root_dir_ = true;
components_.clear();
}
components_.insert(
components_.end(), other.components_.begin(), other.components_.end());
return *this;
}
Path& Path::Concat(const PathString& value) {
auto first_separator = std::find_if(value.begin(), value.end(),
[](PathChar c) {
return std::find(
k_valid_path_separators.begin(),
k_valid_path_separators.end(),
c) != k_valid_path_separators.end();
});
ORT_ENFORCE(first_separator == value.end(),
"Cannot concatenate with a string containing a path separator. String: ", ToUTF8String(value));
if (components_.empty()) {
components_.push_back(value);
} else {
components_.back() += value;
}
return *this;
}
Status RelativePath(const Path& src, const Path& dst, Path& rel) {
ORT_RETURN_IF_NOT(
src.GetRootPathString() == dst.GetRootPathString(),
"Paths must have the same root to compute a relative path. ",
"src root: ", ToUTF8String(src.GetRootPathString()),
", dst root: ", ToUTF8String(dst.GetRootPathString()));
const Path norm_src = src.NormalizedPath(), norm_dst = dst.NormalizedPath();
const auto& src_components = norm_src.GetComponents();
const auto& dst_components = norm_dst.GetComponents();
const auto min_num_components = std::min(
src_components.size(), dst_components.size());
const auto mismatch_point = std::mismatch(
src_components.begin(), src_components.begin() + min_num_components,
dst_components.begin());
const auto& common_src_components_end = mismatch_point.first;
const auto& common_dst_components_end = mismatch_point.second;
std::vector<PathString> rel_components{};
rel_components.reserve(
(src_components.end() - common_src_components_end) +
(dst_components.end() - common_dst_components_end));
std::fill_n(
std::back_inserter(rel_components),
(src_components.end() - common_src_components_end),
k_dotdot);
std::copy(
common_dst_components_end, dst_components.end(),
std::back_inserter(rel_components));
rel = Path(PathString{}, false, rel_components);
return Status::OK();
}
} // namespace onnxruntime

106
onnxruntime/core/common/path.h
View file

@ -1,106 +0,0 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#pragma once
#include <vector>
#include "core/common/common.h"
#include "core/common/path_string.h"
namespace onnxruntime {
#ifdef _WIN32
constexpr PathChar k_preferred_path_separator = ORT_TSTR('\\');
#else // POSIX
constexpr PathChar k_preferred_path_separator = ORT_TSTR('/');
#endif
// Note: We should use the std::filesystem library after upgrading to C++17.
/** A filesystem path. */
class Path {
public:
Path() = default;
Path(const Path&) = default;
Path& operator=(const Path&) = default;
Path(Path&&) = default;
Path& operator=(Path&&) = default;
/** Parses a path from `path_str`. */
static Status Parse(const PathString& path_str, Path& path);
/** Parses a path from `path_str`. Throws on failure. */
static Path Parse(const PathString& path_str);
/** Gets a string representation of the path. */
PathString ToPathString() const;
/** Gets a string representation of the path's root path, if any. */
PathString GetRootPathString() const;
/** Gets the path components following the path root. */
const std::vector<PathString>& GetComponents() const { return components_; }
/** Whether the path is empty. */
bool IsEmpty() const;
/** Whether the path is absolute (refers unambiguously to a file location). */
bool IsAbsolute() const;
/** Whether the path is relative (not absolute). */
bool IsRelative() const { return !IsAbsolute(); }
/** Returns a copy of the path without the last component. */
Path ParentPath() const;
/**
* Normalizes the path.
* A normalized path is one with "."'s and ".."'s resolved.
* Note: This is a pure path computation with no filesystem access.
*/
Path& Normalize();
/** Returns a normalized copy of the path. */
Path NormalizedPath() const {
Path p{*this};
return p.Normalize();
}
/**
* Appends `other` to the path.
* The algorithm should model that of std::filesystem::path::append().
*/
Path& Append(const Path& other);
/**
* Concatenates the current path and the argument string.
* Unlike with Append() or operator/=, additional directory separators are never introduced.
*/
Path& Concat(const PathString& string);
/** Equivalent to this->Append(other). */
Path& operator/=(const Path& other) {
return Append(other);
}
/** Returns `a` appended with `b`. */
friend Path operator/(Path a, const Path& b) {
return a /= b;
}
friend Status RelativePath(const Path& src, const Path& dst, Path& rel);
private:
Path(PathString root_name, bool has_root_dir, std::vector<PathString> components)
: root_name_{std::move(root_name)},
has_root_dir_{has_root_dir},
components_{std::move(components)} {
}
PathString root_name_{};
bool has_root_dir_{false};
std::vector<PathString> components_{};
};
/**
* Computes the relative path from `src` to `dst`.
* Note: This is a pure path computation with no filesystem access.
*/
Status RelativePath(const Path& src, const Path& dst, Path& rel);
} // namespace onnxruntime

10
onnxruntime/core/framework/debug_node_inputs_outputs_utils.cc
View file

@ -76,9 +76,9 @@ PathString MakeTensorFileName(const std::string& tensor_name, const NodeDumpOpti
return path_utils::MakePathString(make_valid_name(tensor_name), dump_options.file_suffix, ".tensorproto");
}
void DumpTensorToFile(const Tensor& tensor, const std::string& tensor_name, const Path& file_path) {
void DumpTensorToFile(const Tensor& tensor, const std::string& tensor_name, const std::filesystem::path& file_path) {
auto tensor_proto = utils::TensorToTensorProto(tensor, tensor_name);
const PathString file_path_str = file_path.ToPathString();
const PathString file_path_str = file_path.native();
int output_fd;
ORT_THROW_IF_ERROR(Env::Default().FileOpenWr(file_path_str, output_fd));
try {
@ -302,7 +302,7 @@ void DumpCpuTensor(
break;
}
case NodeDumpOptions::DataDestination::TensorProtoFiles: {
const Path tensor_file = dump_options.output_dir / Path::Parse(MakeTensorFileName(tensor_metadata.name, dump_options));
const std::filesystem::path tensor_file = dump_options.output_dir / MakeTensorFileName(tensor_metadata.name, dump_options);
DumpTensorToFile(tensor, tensor_metadata.name, tensor_file);
break;
}
@ -411,11 +411,11 @@ const NodeDumpOptions& NodeDumpOptionsFromEnvironmentVariables() {
}
}
opts.output_dir = Path::Parse(ToPathString(Env::Default().GetEnvironmentVar(env_vars::kOutputDir)));
opts.output_dir = ToPathString(Env::Default().GetEnvironmentVar(env_vars::kOutputDir));
std::string sqlite_db_prefix =
ParseEnvironmentVariableWithDefault<std::string>(env_vars::kSqliteDbPrefix, "execution-trace");
opts.sqlite_db_prefix = Path::Parse(ToPathString(sqlite_db_prefix));
opts.sqlite_db_prefix = ToPathString(sqlite_db_prefix);
// check for confirmation for dumping data to files for all nodes
const bool is_input_or_output_requested = ((opts.dump_flags & NodeDumpOptions::DumpFlags::InputData) != 0) ||

5
onnxruntime/core/framework/debug_node_inputs_outputs_utils.h
View file

@ -16,7 +16,6 @@
#pragma once
#include "core/common/path.h"
#include "core/framework/op_kernel.h"
#include "core/framework/session_state.h"
#include "core/graph/graph.h"
@ -109,9 +108,9 @@ struct NodeDumpOptions {
std::string file_suffix;
// the output directory for dumped data files
Path output_dir;
std::filesystem::path output_dir;
// the sqlite3 db to append dumped data
Path sqlite_db_prefix;
std::filesystem::path sqlite_db_prefix;
// Total number of elements which trigger snippet rather than full array for Stdout. Value 0 disables snippet.
int snippet_threshold;

1
onnxruntime/core/framework/tensorprotoutils.h
View file

@ -9,7 +9,6 @@
#ifndef SHARED_PROVIDER
#include "core/common/common.h"
#include "core/common/path.h"
#include "core/common/status.h"
#include "core/common/safeint.h"
#include "core/framework/endian_utils.h"

1
onnxruntime/core/graph/model.h
View file

@ -11,7 +11,6 @@
#include "core/common/flatbuffers.h"
#include "core/common/path.h"
#include "core/graph/graph_viewer.h"
#include "core/graph/ort_format_load_options.h"
#include "core/session/onnxruntime_c_api.h"

1
onnxruntime/core/optimizer/initializer.cc
View file

@ -7,7 +7,6 @@
#include <memory>
#include "core/common/gsl.h"
#include "core/common/path.h"
#include "core/framework/tensorprotoutils.h"
#include "core/framework/tensor_external_data_info.h"
#include "core/platform/env.h"

1
onnxruntime/core/optimizer/initializer.h
View file

@ -10,7 +10,6 @@
#include <filesystem>
#include "core/common/common.h"
#include "core/common/narrow.h"
#include "core/common/path.h"
#include "core/framework/allocator.h"
#include "core/optimizer/graph_transformer.h"
#include "core/framework/tensor_shape.h"

7
onnxruntime/core/providers/shared_library/provider_interfaces.h
View file

@ -903,13 +903,6 @@ struct ProviderHost {
int execution_order) noexcept = 0;
virtual const Node* GraphViewer__GetProducerNode(const GraphViewer* p, const std::string& node_arg_name) const = 0;
// Path
virtual PathString Path__ToPathString(const Path* p) noexcept = 0;
virtual const std::vector<PathString>& Path__GetComponents(const Path* p) noexcept = 0;
virtual bool Path__IsEmpty(const Path* p) noexcept = 0;
virtual std::unique_ptr<Path> Path__construct() = 0;
virtual void Path__operator_delete(ONNX_NAMESPACE::Path* p) = 0;
// OpKernel
virtual const Node& OpKernel__Node(const OpKernel* p) = 0;

13
onnxruntime/core/providers/shared_library/provider_wrappedtypes.h
View file

@ -968,19 +968,6 @@ class GraphViewer final {
void operator=(const GraphViewer&) = delete;
};
struct Path final {
static std::unique_ptr<Path> Create() { return g_host->Path__construct(); }
static void operator delete(void* p) { g_host->Path__operator_delete(reinterpret_cast<Path*>(p)); }
PathString ToPathString() const noexcept { return g_host->Path__ToPathString(this); }
const std::vector<PathString>& GetComponents() const noexcept { return g_host->Path__GetComponents(this); }
bool IsEmpty() const noexcept { return g_host->Path__IsEmpty(this); }
Path() = delete;
Path(const Path&) = delete;
void operator=(const Path&) = delete;
};
struct OpKernelContext final {
template <typename T>
const T& RequiredInput(int index) const;

1
onnxruntime/core/providers/tvm/tvm_api.cc
View file

@ -16,7 +16,6 @@
#include <tvm/target/codegen.h>
#include "core/common/common.h"
#include "core/common/path.h"
#include "core/common/gsl.h"
#include "tvm_api.h"

3
onnxruntime/core/providers/vitisai/imp/tensor_proto.cc
View file

@ -12,8 +12,7 @@ gsl::span<const char> tensor_proto_as_raw(const ONNX_NAMESPACE::TensorProto& ten
auto& mut_tensor = const_cast<ONNX_NAMESPACE::TensorProto&>(tensor);
if (!tensor.has_raw_data()) {
std::vector<uint8_t> unpacked_tensor;
auto path = onnxruntime::Path::Create();
auto s = onnxruntime::utils::UnpackInitializerData(tensor, *path, unpacked_tensor);
auto s = onnxruntime::utils::UnpackInitializerData(tensor, std::filesystem::path(), unpacked_tensor);
mut_tensor.mutable_raw_data()->resize(unpacked_tensor.size());
mut_tensor.clear_float_data();
mut_tensor.clear_int32_data();

8
onnxruntime/core/session/provider_bridge_ort.cc
View file

@ -5,6 +5,7 @@
// It implements onnxruntime::ProviderHost
#include "core/common/inlined_containers.h"
#include "core/common/path_string.h"
#include "core/framework/allocator_utils.h"
#include "core/framework/config_options.h"
#include "core/framework/compute_capability.h"
@ -1203,13 +1204,6 @@ struct ProviderHostImpl : ProviderHost {
}
const Node* GraphViewer__GetProducerNode(const GraphViewer* p, const std::string& node_arg_name) const override { return p->GetProducerNode(node_arg_name); }
// Path (wrapped)
PathString Path__ToPathString(const Path* p) noexcept override { return p->ToPathString(); }
const std::vector<PathString>& Path__GetComponents(const Path* p) noexcept override { return p->GetComponents(); }
bool Path__IsEmpty(const Path* p) noexcept override { return p->IsEmpty(); }
std::unique_ptr<Path> Path__construct() override { return std::make_unique<Path>(); }
void Path__operator_delete(ONNX_NAMESPACE::Path* p) override { delete p; };
// OpKernel (direct)
const Node& OpKernel__Node(const OpKernel* p) override { return p->OpKernel::Node(); }

256
onnxruntime/test/common/path_test.cc
View file

@ -1,256 +0,0 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "core/common/path.h"
#include "gtest/gtest.h"
#include "core/common/optional.h"
#include "test/util/include/asserts.h"
namespace onnxruntime {
namespace test {
TEST(PathTest, Parse) {
auto check_parse =
[](const std::string& path_string,
const std::string& expected_root,
const std::vector<std::string>& expected_components) {
Path p{};
ASSERT_STATUS_OK(Path::Parse(ToPathString(path_string), p));
std::vector<PathString> expected_components_ps{};
std::transform(
expected_components.begin(), expected_components.end(),
std::back_inserter(expected_components_ps),
[](const std::string& s) { return ToPathString(s); });
EXPECT_EQ(p.GetComponents(), expected_components_ps);
EXPECT_EQ(p.GetRootPathString(), ToPathString(expected_root));
};
check_parse(
"i/am/relative",
"", {"i", "am", "relative"});
#ifdef _WIN32
check_parse(
"/i/am/rooted",
R"(\)", {"i", "am", "rooted"});
check_parse(
R"(\\server\share\i\am\rooted)",
R"(\\server\share\)", {"i", "am", "rooted"});
check_parse(
R"(C:\i\am\rooted)",
R"(C:\)", {"i", "am", "rooted"});
check_parse(
R"(C:i\am\relative)",
"C:", {"i", "am", "relative"});
#else // POSIX
check_parse(
"/i/am/rooted",
"/", {"i", "am", "rooted"});
check_parse(
"//root_name/i/am/rooted",
"//root_name/", {"i", "am", "rooted"});
#endif
}
TEST(PathTest, ParseFailure) {
auto check_parse_failure =
[](const std::string& path_string) {
Path p{};
EXPECT_FALSE(Path::Parse(ToPathString(path_string), p).IsOK());
};
#ifdef _WIN32
check_parse_failure(R"(\\server_name_no_separator)");
check_parse_failure(R"(\\server_name_no_share_name\)");
check_parse_failure(R"(\\server_name\share_name_no_root_dir)");
#else // POSIX
check_parse_failure("//root_name_no_root_dir");
#endif
}
TEST(PathTest, IsEmpty) {
auto check_empty =
[](const std::string& path_string, bool is_empty) {
Path p{};
ASSERT_STATUS_OK(Path::Parse(ToPathString(path_string), p));
EXPECT_EQ(p.IsEmpty(), is_empty);
};
check_empty("", true);
check_empty(".", false);
check_empty("/", false);
}
TEST(PathTest, IsAbsoluteOrRelative) {
auto check_abs_or_rel =
[](const std::string& path_string, bool is_absolute) {
Path p{};
ASSERT_STATUS_OK(Path::Parse(ToPathString(path_string), p));
EXPECT_EQ(p.IsAbsolute(), is_absolute);
EXPECT_EQ(p.IsRelative(), !is_absolute);
};
check_abs_or_rel("relative", false);
check_abs_or_rel("", false);
#ifdef _WIN32
check_abs_or_rel(R"(\root_relative)", false);
check_abs_or_rel(R"(\)", false);
check_abs_or_rel("C:drive_relative", false);
check_abs_or_rel("C:", false);
check_abs_or_rel(R"(C:\absolute)", true);
check_abs_or_rel(R"(C:\)", true);
#else // POSIX
check_abs_or_rel("/absolute", true);
check_abs_or_rel("/", true);
#endif
}
TEST(PathTest, ParentPath) {
auto check_parent =
[](const std::string path_string, const std::string& expected_parent_path_string) {
Path p{}, p_expected_parent{};
ASSERT_STATUS_OK(Path::Parse(ToPathString(path_string), p));
ASSERT_STATUS_OK(Path::Parse(ToPathString(expected_parent_path_string), p_expected_parent));
EXPECT_EQ(p.ParentPath().ToPathString(), p_expected_parent.ToPathString());
};
check_parent("a/b", "a");
check_parent("/a/b", "/a");
check_parent("", "");
check_parent("/", "/");
}
TEST(PathTest, Normalize) {
auto check_normalize =
[](const std::string& path_string,
const std::string& expected_normalized_path_string) {
Path p{}, p_expected_normalized{};
ASSERT_STATUS_OK(Path::Parse(ToPathString(path_string), p));
ASSERT_STATUS_OK(Path::Parse(ToPathString(expected_normalized_path_string), p_expected_normalized));
EXPECT_EQ(p.Normalize().ToPathString(), p_expected_normalized.ToPathString());
};
check_normalize("/a/b/./c/../../d/../e", "/a/e");
check_normalize("a/b/./c/../../d/../e", "a/e");
check_normalize("/../a/../../b", "/b");
check_normalize("../a/../../b", "../../b");
check_normalize("/a/..", "/");
check_normalize("a/..", ".");
check_normalize("", "");
check_normalize("/", "/");
check_normalize(".", ".");
}
TEST(PathTest, Append) {
auto check_append =
[](const std::string& a, const std::string& b, const std::string& expected_ab) {
Path p_a{}, p_b{}, p_expected_ab{};
ASSERT_STATUS_OK(Path::Parse(ToPathString(a), p_a));
ASSERT_STATUS_OK(Path::Parse(ToPathString(b), p_b));
ASSERT_STATUS_OK(Path::Parse(ToPathString(expected_ab), p_expected_ab));
EXPECT_EQ(p_a.Append(p_b).ToPathString(), p_expected_ab.ToPathString());
};
check_append("/a/b", "c/d", "/a/b/c/d");
check_append("/a/b", "/c/d", "/c/d");
check_append("a/b", "c/d", "a/b/c/d");
check_append("a/b", "/c/d", "/c/d");
#ifdef _WIN32
check_append(R"(C:\a\b)", R"(c\d)", R"(C:\a\b\c\d)");
check_append(R"(C:\a\b)", R"(\c\d)", R"(C:\c\d)");
check_append(R"(C:\a\b)", R"(D:c\d)", R"(D:c\d)");
check_append(R"(C:\a\b)", R"(D:\c\d)", R"(D:\c\d)");
check_append(R"(C:a\b)", R"(c\d)", R"(C:a\b\c\d)");
check_append(R"(C:a\b)", R"(\c\d)", R"(C:\c\d)");
check_append(R"(C:a\b)", R"(D:c\d)", R"(D:c\d)");
check_append(R"(C:a\b)", R"(D:\c\d)", R"(D:\c\d)");
#else // POSIX
check_append("//root_0/a/b", "c/d", "//root_0/a/b/c/d");
check_append("//root_0/a/b", "/c/d", "/c/d");
check_append("//root_0/a/b", "//root_1/c/d", "//root_1/c/d");
#endif
}
TEST(PathTest, RelativePath) {
auto check_relative =
[](const std::string& src,
const std::string& dst,
const std::string& expected_rel) {
Path p_src, p_dst, p_expected_rel, p_rel;
ASSERT_STATUS_OK(Path::Parse(ToPathString(src), p_src));
ASSERT_STATUS_OK(Path::Parse(ToPathString(dst), p_dst));
ASSERT_STATUS_OK(Path::Parse(ToPathString(expected_rel), p_expected_rel));
ASSERT_STATUS_OK(RelativePath(p_src, p_dst, p_rel));
EXPECT_EQ(p_rel.ToPathString(), p_expected_rel.ToPathString());
};
check_relative(
"/a/b/c/d/e", "/a/b/c/d/e/f/g/h",
"f/g/h");
check_relative(
"/a/b/c/d/e", "/a/b/f/g/h/i",
"../../../f/g/h/i");
check_relative(
"a/b/../c/../d", "e/./f/../g/h",
"../../e/g/h");
}
TEST(PathTest, RelativePathFailure) {
auto check_relative_failure =
[](const std::string& src,
const std::string& dst) {
Path p_src, p_dst, p_rel;
ASSERT_STATUS_OK(Path::Parse(ToPathString(src), p_src));
ASSERT_STATUS_OK(Path::Parse(ToPathString(dst), p_dst));
EXPECT_FALSE(RelativePath(p_src, p_dst, p_rel).IsOK());
};
check_relative_failure("/rooted", "relative");
check_relative_failure("relative", "/rooted");
#ifdef _WIN32
check_relative_failure("C:/a", "D:/a");
#else // POSIX
check_relative_failure("//root_0/a", "//root_1/a");
#endif
}
#if !defined(ORT_NO_EXCEPTIONS)
TEST(PathTest, Concat) {
auto check_concat =
[](const optional<std::string>& a, const std::string& b, const std::string& expected_a, bool expect_throw = false) {
Path p_a{}, p_expected_a{};
if (a.has_value()) {
ASSERT_STATUS_OK(Path::Parse(ToPathString(*a), p_a));
}
ASSERT_STATUS_OK(Path::Parse(ToPathString(expected_a), p_expected_a));
if (expect_throw) {
EXPECT_THROW(p_a.Concat(ToPathString(b)).ToPathString(), OnnxRuntimeException);
} else {
EXPECT_EQ(p_a.Concat(ToPathString(b)).ToPathString(), p_expected_a.ToPathString());
}
};
check_concat({"/a/b"}, "c", "/a/bc");
check_concat({"a/b"}, "cd", "a/bcd");
check_concat({""}, "cd", "cd");
check_concat({}, "c", "c");
#ifdef _WIN32
check_concat({"a/b"}, R"(c\d)", "", true /* expect_throw */);
#else
check_concat({"a/b"}, "c/d", "", true /* expect_throw */);
#endif
}
#endif
} // namespace test
} // namespace onnxruntime

4
onnxruntime/test/debug_node_inputs_outputs/debug_node_inputs_outputs_utils_test.cc
View file

@ -27,7 +27,7 @@ void VerifyTensorProtoFileData(const PathString& tensor_proto_path, gsl::span<co
std::vector<T> actual_data{};
actual_data.resize(expected_data.size());
ASSERT_STATUS_OK(utils::UnpackTensor(tensor_proto, Path{}, actual_data.data(), actual_data.size()));
ASSERT_STATUS_OK(utils::UnpackTensor(tensor_proto, tensor_proto_path, actual_data.data(), actual_data.size()));
ASSERT_EQ(gsl::span<const T>(actual_data), expected_data);
}
@ -48,7 +48,7 @@ void VerifyTensorProtoFileDataInt4(const PathString& tensor_proto_path,
std::vector<Int4x2Base<Signed>> actual_data{};
actual_data.resize(expected_data.size());
ASSERT_STATUS_OK(utils::UnpackTensor(tensor_proto, Path{}, actual_data.data(), num_elems));
ASSERT_STATUS_OK(utils::UnpackTensor(tensor_proto, tensor_proto_path, actual_data.data(), num_elems));
ASSERT_EQ(actual_data.size(), expected_data.size());

1
onnxruntime/test/flatbuffers/flatbuffer_utils_test.cc
View file

@ -9,7 +9,6 @@
#include "gtest/gtest.h"
#include "core/common/common.h"
#include "core/common/path.h"
#include "core/graph/graph_flatbuffers_utils.h"
#include "core/framework/tensorprotoutils.h"
#include "core/providers/cpu/cpu_execution_provider.h"

8
onnxruntime/test/ir/graph_test.cc
View file

@ -1782,8 +1782,8 @@ TEST_F(GraphTest, InjectExternalInitializedTensors) {
{initializer_name, ort_value}};
// We do not need actual files there since we are not going to load it.
const auto tensor_data_dir_path = Path::Parse(ToPathString("."));
const auto tensor_data_dir_relative_path = Path::Parse(ToPathString("external_data.bin"));
const auto tensor_data_dir_path = ORT_TSTR(".");
const auto tensor_data_dir_relative_path = ORT_TSTR("external_data.bin");
const auto tensor_proto =
[&]() {
@ -1792,7 +1792,7 @@ TEST_F(GraphTest, InjectExternalInitializedTensors) {
tensor_proto.add_dims(tensor_data.size());
tensor_proto.set_data_type(ONNX_NAMESPACE::TensorProto_DataType_INT32);
tensor_proto.set_data_location(ONNX_NAMESPACE::TensorProto_DataLocation_EXTERNAL);
SetTensorProtoExternalData("location", ToUTF8String(tensor_data_dir_relative_path.ToPathString()),
SetTensorProtoExternalData("location", ToUTF8String(tensor_data_dir_relative_path),
tensor_proto);
SetTensorProtoExternalData("offset", "0", tensor_proto);
SetTensorProtoExternalData("length", std::to_string(tensor_data.size() * sizeof(int32_t)), tensor_proto);
@ -1827,7 +1827,7 @@ TEST_F(GraphTest, InjectExternalInitializedTensors) {
ASSERT_FALSE(utils::HasExternalData(*with_data));
const auto& original_tensor = ort_value.Get<Tensor>();
Tensor replaced_tensor(original_tensor.DataType(), data_shape, std::make_shared<CPUAllocator>());
ASSERT_STATUS_OK(utils::TensorProtoToTensor(Env::Default(), tensor_data_dir_path.ToPathString().c_str(), *with_data,
ASSERT_STATUS_OK(utils::TensorProtoToTensor(Env::Default(), tensor_data_dir_path, *with_data,
replaced_tensor));
ASSERT_EQ(original_tensor.GetElementType(), replaced_tensor.GetElementType());
const auto original_span = original_tensor.DataAsSpan<int32_t>();

7
onnxruntime/test/optimizer/graph_transform_test.cc
View file

@ -6215,9 +6215,10 @@ TEST_F(GraphTransformationTests, PropagateCastOpsTests) {
std::make_unique<PropagateCastOps>(strategy, level, test_case.allow_ops),
TransformerLevel::Level1));
ASSERT_STATUS_OK(graph_transformation_mgr.ApplyTransformers(graph, TransformerLevel::Level1, *logger_));
Path p = Path::Parse(test_case.model_uri);
ASSERT_FALSE(p.GetComponents().empty());
PathString transformed_model_uri = temp_dir.Path() + GetPathSep<PathChar>() + ORT_TSTR("transformed_") + p.GetComponents().back();
std::filesystem::path p = test_case.model_uri;
PathString model_filename = ORT_TSTR("transformed_");
model_filename += p.filename();
std::filesystem::path transformed_model_uri = std::filesystem::path(temp_dir.Path()) / model_filename;
ASSERT_STATUS_OK(Model::Save(*p_model, transformed_model_uri));
// Load the transformed model to validate
ASSERT_STATUS_OK(Model::Load(transformed_model_uri, p_model, nullptr, *logger_));

1
orttraining/orttraining/core/framework/checkpoint_common.h
View file

@ -6,7 +6,6 @@
#include "core/framework/tensorprotoutils.h"
#include "core/common/logging/logging.h"
#include "core/common/logging/sinks/clog_sink.h"
#include "core/common/path.h"
#include "core/common/path_string.h"
#include "core/common/status.h"
#include "core/framework/framework_common.h"

10
orttraining/orttraining/core/framework/checkpointing.cc
View file

@ -10,7 +10,6 @@
#include "core/common/common.h"
#include "core/common/logging/logging.h"
#include "core/common/path.h"
#include "core/framework/data_transfer_utils.h"
#include "core/framework/endian_utils.h"
#include "core/framework/ort_value.h"
@ -260,13 +259,10 @@ Status LoadModelCheckpoint(
ORT_RETURN_IF_ERROR(Env::Default().GetCanonicalPath(
checkpoint_path, checkpoint_canonical_path));
Path relative_tensors_data_path_obj{};
ORT_RETURN_IF_ERROR(RelativePath(
Path::Parse(model_directory_canonical_path),
Path::Parse(GetCheckpointTensorsDataFilePath(checkpoint_canonical_path)),
relative_tensors_data_path_obj));
std::filesystem::path relative_tensors_data_path_obj = std::filesystem::relative(
GetCheckpointTensorsDataFilePath(checkpoint_canonical_path), model_directory_canonical_path);
ORT_RETURN_IF_ERROR(UpdateTensorsExternalDataLocations(
relative_tensors_data_path_obj.ToPathString(), loaded_tensor_protos));
relative_tensors_data_path_obj.native(), loaded_tensor_protos));
}
// read properties file

5
orttraining/orttraining/models/runner/training_runner.cc
View file

@ -1019,9 +1019,8 @@ Status TrainingRunner::SavePerfMetrics(const size_t number_of_batches, const siz
optimizer = optimizer.substr(0, pos);
perf_metrics["Optimizer"] = optimizer;
Path model_path{};
ORT_RETURN_IF_ERROR(Path::Parse(params_.model_path, model_path));
PathString leaf = model_path.GetComponents().back();
std::filesystem::path model_path = params_.model_path;
PathString leaf = model_path.filename();
std::string model_name = ToUTF8String(leaf.c_str());
perf_metrics["ModelName"] = model_name;