From 4d44d8c00a0b20b3e9a6723c665bd45172f4a41b Mon Sep 17 00:00:00 2001 From: PyTorch MergeBot Date: Mon, 18 Sep 2023 18:41:12 +0000 Subject: [PATCH] Revert "Eliminate c10::stoi,c10::stod,c10::stoull,c10::stoll (#109179)" This reverts commit 852f1b8417e80b72a7d1c4a772f66af28da02913. Reverted https://github.com/pytorch/pytorch/pull/109179 on behalf of https://github.com/huydhn due to Sorry for reverting your change but this is breaking periodic buck build, so please fix the issue and reland the change https://github.com/pytorch/pytorch/actions/runs/6207458526/job/16852695272 ([comment](https://github.com/pytorch/pytorch/pull/109179#issuecomment-1724168571)) --- aten/src/ATen/ParallelCommon.cpp | 2 +- aten/src/ATen/core/type.cpp | 2 +- c10/core/Device.cpp | 2 +- c10/util/string_utils.h | 111 +++++++++++++++++- caffe2/core/common_test.cc | 6 +- caffe2/core/net_async_tracing.cc | 2 +- caffe2/opt/mobile.cc | 2 +- caffe2/python/pybind_state.cc | 2 +- caffe2/serialize/inline_container.cc | 2 +- torch/csrc/distributed/rpc/utils.cpp | 2 +- .../jit/frontend/function_schema_parser.cpp | 8 +- torch/csrc/jit/frontend/name_mangler.cpp | 2 +- .../csrc/jit/frontend/schema_type_parser.cpp | 8 +- torch/csrc/jit/frontend/tree_views.h | 2 +- torch/csrc/jit/ir/ir.cpp | 2 +- torch/csrc/jit/ir/irparser.cpp | 6 +- torch/csrc/jit/jit_opt_limit.cpp | 2 +- torch/csrc/jit/mobile/nnc/aot_compiler.cpp | 4 +- torch/csrc/jit/runtime/register_prim_ops.cpp | 4 +- torch/csrc/jit/testing/file_check.cpp | 2 +- 20 files changed, 138 insertions(+), 35 deletions(-) diff --git a/aten/src/ATen/ParallelCommon.cpp b/aten/src/ATen/ParallelCommon.cpp index 285713417cb..7f9284697d2 100644 --- a/aten/src/ATen/ParallelCommon.cpp +++ b/aten/src/ATen/ParallelCommon.cpp @@ -28,7 +28,7 @@ const char* get_env_var( size_t get_env_num_threads(const char* var_name, size_t def_value = 0) { try { if (auto* value = std::getenv(var_name)) { - int nthreads = std::stoi(value); + int nthreads = c10::stoi(value); TORCH_CHECK(nthreads > 0); return nthreads; } diff --git a/aten/src/ATen/core/type.cpp b/aten/src/ATen/core/type.cpp index 3ecfab9fee5..56916b06cfa 100644 --- a/aten/src/ATen/core/type.cpp +++ b/aten/src/ATen/core/type.cpp @@ -47,7 +47,7 @@ static_assert( TypeVerbosity type_verbosity() { static const char* c_verbosity = std::getenv("PYTORCH_JIT_TYPE_VERBOSITY"); static TypeVerbosity verbosity = c_verbosity ? - static_cast(std::stoi(c_verbosity)) : TypeVerbosity::Default; + static_cast(c10::stoi(c_verbosity)) : TypeVerbosity::Default; return verbosity; } diff --git a/c10/core/Device.cpp b/c10/core/Device.cpp index 615383c9e89..032b3329763 100644 --- a/c10/core/Device.cpp +++ b/c10/core/Device.cpp @@ -127,7 +127,7 @@ Device::Device(const std::string& device_string) : Device(Type::CPU) { try { if (!device_index_str.empty()) { - index_ = static_cast(std::stoi(device_index_str)); + index_ = static_cast(c10::stoi(device_index_str)); } } catch (const std::exception&) { TORCH_CHECK( diff --git a/c10/util/string_utils.h b/c10/util/string_utils.h index 167c03b2727..e6a9685bf89 100644 --- a/c10/util/string_utils.h +++ b/c10/util/string_utils.h @@ -18,17 +18,120 @@ std::string to_string(T value) { return os.str(); } +inline int stoi(const std::string& str, std::size_t* pos = 0) { + std::stringstream ss; + int n = 0; + ss << str; + ss >> n; + if (ss.fail()) { + // To mimic `std::stoi` and to avoid including `Exception.h`, throw + // `std::invalid_argument`. + // We can't easily detect out-of-range, so we don't use `std::out_of_range`. + throw std::invalid_argument("Not an integer"); + } + if (pos) { + if (ss.tellg() == std::streampos(-1)) { + *pos = str.size(); + } else { + *pos = ss.tellg(); + } + } + return n; +} + +inline uint64_t stoull(const std::string& str) { + std::stringstream ss; + uint64_t n = 0; + ss << str; + ss >> n; + if (ss.fail()) { + // To mimic `std::stoull` and to avoid including `Exception.h`, throw + // `std::invalid_argument`. + // We can't easily detect out-of-range, so we don't use `std::out_of_range`. + throw std::invalid_argument("Not an unsigned 64-bit integer"); + } + return n; +} + +inline double stod(const std::string& str, std::size_t* pos = 0) { + std::stringstream ss; + ss << str; + double val = 0; + ss >> val; + if (ss.fail()) { + // To mimic `std::stod` and to avoid including `Exception.h`, throw + // `std::invalid_argument`. + // We can't easily detect out-of-range, so we don't use `std::out_of_range`. + throw std::invalid_argument("Not a double-precision floating point number"); + } + if (pos) { + if (ss.tellg() == std::streampos(-1)) { + *pos = str.size(); + } else { + *pos = ss.tellg(); + } + } + return val; +} + +inline long long stoll(const std::string& str, std::size_t* pos = 0) { + // std::stoll doesn't exist in our Android environment, we need to implement + // it ourselves. + std::stringstream ss; + ss << str; + long long result = 0; + ss >> result; + if (ss.fail()) { + // To mimic `std::stoll` and to avoid including `Exception.h`, throw + // `std::invalid_argument`. + // We can't easily detect out-of-range, so we don't use `std::out_of_range`. + throw std::invalid_argument("Not a long long integer"); + } + if (pos) { + if (ss.tellg() == std::streampos(-1)) { + *pos = str.size(); + } else { + *pos = ss.tellg(); + } + } + return result; +} + +inline long long stoll(const std::string& str, size_t pos, int base) { + // std::stoll doesn't exist in our Android environment, we need to implement + // it ourselves. + std::stringstream ss; + if (str.size() > 0 && str.at(0) == '0') { + if (str.size() > 1 && (str.at(1) == 'x' || str.at(1) == 'X')) { + ss << std::hex << str; + } else { + ss << std::oct << str; + } + } else { + ss << str; + } + long long result = 0; + ss >> result; + if (ss.fail()) { + // To mimic `std::stoll` and to avoid including `Exception.h`, throw + // `std::invalid_argument`. + // We can't easily detect out-of-range, so we don't use `std::out_of_range`. + throw std::invalid_argument("Not a long long integer"); + } + return result; +} + #else #define CAFFE2_TESTONLY_WE_ARE_USING_CUSTOM_STRING_FUNCTIONS 0 +using std::stod; +using std::stoi; +using std::stoll; +using std::stoull; using std::to_string; #endif // defined(__ANDROID__) || defined(CAFFE2_FORCE_STD_STRING_FALLBACK_TEST) } // namespace c10 -using std::stod; -using std::stoi; -using std::stoll; -using std::stoull; #if defined(__ANDROID__) && __ANDROID_API__ < 21 && defined(__GLIBCXX__) #include // std::strtoll isn't available on Android NDK platform < 21 when building diff --git a/caffe2/core/common_test.cc b/caffe2/core/common_test.cc index e28e2c7c0e4..7de0a737099 100644 --- a/caffe2/core/common_test.cc +++ b/caffe2/core/common_test.cc @@ -17,7 +17,7 @@ TEST(CommonTest, TestStoi) { EXPECT_TRUE(CAFFE2_TESTONLY_WE_ARE_USING_CUSTOM_STRING_FUNCTIONS); string s = "1234"; int i_std = std::stoi(s); - int i_caffe2 = ::std::stoi(s); + int i_caffe2 = ::c10::stoi(s); EXPECT_EQ(i_std, i_caffe2); } @@ -26,14 +26,14 @@ TEST(CommonTest, TestStod) { string s = "1.234"; std::size_t p_std = 0, p_caffe2 = 0; double d_std = std::stod(s, &p_std); - double d_caffe2 = ::std::stod(s, &p_caffe2); + double d_caffe2 = ::c10::stod(s, &p_caffe2); EXPECT_EQ(d_std, d_caffe2); EXPECT_EQ(p_std, p_caffe2); // Only part of the string is parsed. s = "1.234 5.678"; d_std = std::stod(s, &p_std); - d_caffe2 = ::std::stod(s, &p_caffe2); + d_caffe2 = ::c10::stod(s, &p_caffe2); EXPECT_EQ(d_std, d_caffe2); EXPECT_EQ(p_std, p_caffe2); } diff --git a/caffe2/core/net_async_tracing.cc b/caffe2/core/net_async_tracing.cc index 93396c37d15..a571b38dcef 100644 --- a/caffe2/core/net_async_tracing.cc +++ b/caffe2/core/net_async_tracing.cc @@ -402,7 +402,7 @@ int extractShardId(const std::string& name) { while (right_pos < name.length() && isdigit(name[right_pos])) { right_pos++; } - return std::stoi(name.substr(left_pos, right_pos - left_pos)); + return c10::stoi(name.substr(left_pos, right_pos - left_pos)); } else { return -1; } diff --git a/caffe2/opt/mobile.cc b/caffe2/opt/mobile.cc index 768247a9852..c54b70405f0 100644 --- a/caffe2/opt/mobile.cc +++ b/caffe2/opt/mobile.cc @@ -98,7 +98,7 @@ void fuseNNPACKConvRelu(repr::NNModule* nn) { if (op.engine() != "NNPACK") { return false; } - std::string algo = "AUTO"; + caffe2::string algo = "AUTO"; for (const auto &arg : op.arg()) { if (arg.name() == "algo") { algo = arg.s(); diff --git a/caffe2/python/pybind_state.cc b/caffe2/python/pybind_state.cc index 9b71e668196..f24a5da35af 100644 --- a/caffe2/python/pybind_state.cc +++ b/caffe2/python/pybind_state.cc @@ -192,7 +192,7 @@ py::object fetchBlob(Workspace* ws, const std::string& name) { // If there is no fetcher registered, return a metainfo string. // If all branches failed, we will return a metainfo string. std::stringstream ss; - ss << std::string(name) << ", a C++ native class of type " + ss << caffe2::string(name) << ", a C++ native class of type " << blob.TypeName() << "."; return py::bytes(ss.str()); } diff --git a/caffe2/serialize/inline_container.cc b/caffe2/serialize/inline_container.cc index fb763a89f00..e527f883ee0 100644 --- a/caffe2/serialize/inline_container.cc +++ b/caffe2/serialize/inline_container.cc @@ -192,7 +192,7 @@ void PyTorchStreamReader::init() { } std::string version(static_cast(version_ptr.get()), version_size); try { - version_ = std::stoull(version); + version_ = caffe2::stoull(version); } catch (const std::invalid_argument& e) { CAFFE_THROW("Couldn't parse the version ", version, diff --git a/torch/csrc/distributed/rpc/utils.cpp b/torch/csrc/distributed/rpc/utils.cpp index 822079b12ec..ee295a55335 100644 --- a/torch/csrc/distributed/rpc/utils.cpp +++ b/torch/csrc/distributed/rpc/utils.cpp @@ -297,7 +297,7 @@ parseWireSections(const void* data, size_t data_size) { if (ptr == endp) { break; } - size_t sz = std::stoll(std::string(sizePtr, ptr - sizePtr)); + size_t sz = c10::stoll(std::string(sizePtr, ptr - sizePtr)); headerEnts.emplace_back(name, sz); ++ptr; // past the '\n' } diff --git a/torch/csrc/jit/frontend/function_schema_parser.cpp b/torch/csrc/jit/frontend/function_schema_parser.cpp index 4d74ec4714c..8df74f3c2f4 100644 --- a/torch/csrc/jit/frontend/function_schema_parser.cpp +++ b/torch/csrc/jit/frontend/function_schema_parser.cpp @@ -161,7 +161,7 @@ struct SchemaParser { // note: an array with a size hint can only occur at the Argument level fake_type = ListType::create(std::move(fake_type)); real_type = ListType::create(std::move(real_type)); - N = std::stoll(L.expect(TK_NUMBER).text()); + N = c10::stoll(L.expect(TK_NUMBER).text()); L.expect(']'); auto container = type_parser.parseAliasAnnotation(); if (alias_info) { @@ -248,14 +248,14 @@ struct SchemaParser { n = L.expect(TK_NUMBER).text(); if (kind == TypeKind::ComplexType || n.find('j') != std::string::npos) { - auto imag = std::stod(n.substr(0, n.size() - 1)); + auto imag = c10::stod(n.substr(0, n.size() - 1)); return c10::complex(0, imag); } else if ( kind == TypeKind::FloatType || n.find('.') != std::string::npos || n.find('e') != std::string::npos) { - return std::stod(n); + return c10::stod(n); } else { - int64_t v = std::stoll(n); + int64_t v = c10::stoll(n); return v; } } diff --git a/torch/csrc/jit/frontend/name_mangler.cpp b/torch/csrc/jit/frontend/name_mangler.cpp index fbf1d24932e..0f5daf12faa 100644 --- a/torch/csrc/jit/frontend/name_mangler.cpp +++ b/torch/csrc/jit/frontend/name_mangler.cpp @@ -13,7 +13,7 @@ c10::QualifiedName NameMangler::mangle(const c10::QualifiedName& name) { if (pos != std::string::npos) { auto num = atom.substr(pos + manglePrefix.size()); // current mangle index in the name - size_t num_i = std::stoi(num); + size_t num_i = c10::stoi(num); // bump the mangleIndex_ to num_i + 1 mangleIndex_ = std::max(mangleIndex_, num_i + 1); std::string newAtomPrefix; diff --git a/torch/csrc/jit/frontend/schema_type_parser.cpp b/torch/csrc/jit/frontend/schema_type_parser.cpp index 7c4b8ba0cac..77be2adc425 100644 --- a/torch/csrc/jit/frontend/schema_type_parser.cpp +++ b/torch/csrc/jit/frontend/schema_type_parser.cpp @@ -177,7 +177,7 @@ c10::optional SchemaTypeParser::tryToParseDeviceType() { // NOLINTNEXTLINE(cppcoreguidelines-init-variables) std::string::size_type num_len; try { - device_idx = std::stoi(num, &num_len); + device_idx = c10::stoi(num, &num_len); } catch (const std::invalid_argument& e) { throw ErrorReport(L.cur()) << "Device index cannot be converted to integer"; @@ -202,7 +202,7 @@ c10::optional SchemaTypeParser::tryToParseRequiresGrad() { std::string::size_type num_len; try { - return (bool)std::stoi(num, &num_len); + return (bool)c10::stoi(num, &num_len); } catch (const std::invalid_argument& e) { throw ErrorReport(L.cur()) << "Field requires_grad cannot be converted to integer"; @@ -262,7 +262,7 @@ TypePtr SchemaTypeParser::parseRefinedTensor() { // NOLINTNEXTLINE(cppcoreguidelines-init-variables) std::string::size_type num_len; try { - auto stride = std::stoll(num, &num_len); + auto stride = c10::stoll(num, &num_len); strides.push_back(stride); } catch (const std::invalid_argument& e) { throw ErrorReport(L.cur()) @@ -302,7 +302,7 @@ TypePtr SchemaTypeParser::parseRefinedTensor() { std::string::size_type num_len; int64_t dim = 0; try { - dim = std::stoll(num, &num_len); + dim = c10::stoll(num, &num_len); } catch (const std::invalid_argument& e) { throw ErrorReport(L.cur()) << "The number can't be converted to int"; } catch (const std::out_of_range& e) { diff --git a/torch/csrc/jit/frontend/tree_views.h b/torch/csrc/jit/frontend/tree_views.h index 56100215f9b..22adb275d5c 100644 --- a/torch/csrc/jit/frontend/tree_views.h +++ b/torch/csrc/jit/frontend/tree_views.h @@ -907,7 +907,7 @@ struct Const : public Expr { int64_t asIntegral() const { try { // NOLINTNEXTLINE(modernize-use-nullptr) - return std::stoll(subtree(0)->stringValue(), /*__idx=*/0, /*base=*/0); + return c10::stoll(subtree(0)->stringValue(), /*__idx=*/0, /*base=*/0); } catch (const std::out_of_range&) { throw ErrorReport(range()) << "Integral constant out of range " "(must fit in a signed 64 bit integer)"; diff --git a/torch/csrc/jit/ir/ir.cpp b/torch/csrc/jit/ir/ir.cpp index 4ebb0df7535..852dce474bf 100644 --- a/torch/csrc/jit/ir/ir.cpp +++ b/torch/csrc/jit/ir/ir.cpp @@ -876,7 +876,7 @@ Value* Value::setDebugName(const std::string& name) { if (last_dot_pos != std::string::npos && last_dot_pos + 1 != name.size()) { if (name.find_first_not_of("0123456789", last_dot_pos + 1) == std::string::npos) { - suffix = std::stoll(name.substr(last_dot_pos + 1)); + suffix = c10::stoll(name.substr(last_dot_pos + 1)); name_base = name.substr(0, last_dot_pos); } } diff --git a/torch/csrc/jit/ir/irparser.cpp b/torch/csrc/jit/ir/irparser.cpp index c37988e322a..0f86c7af1e5 100644 --- a/torch/csrc/jit/ir/irparser.cpp +++ b/torch/csrc/jit/ir/irparser.cpp @@ -191,7 +191,7 @@ ParsedLiteral IRParser::parseScalarLiteral(Node* n) { r.k = AttributeKind::c; double imag = 0.0f; try { - imag = std::stod(str.substr(0, str.size() - 1)); + imag = c10::stod(str.substr(0, str.size() - 1)); } catch (const std::invalid_argument& e) { throw ErrorReport(token.range) << "Number cannot be converted to double"; @@ -205,7 +205,7 @@ ParsedLiteral IRParser::parseScalarLiteral(Node* n) { str.find('e') != std::string::npos) { r.k = AttributeKind::f; try { - r.f = std::stod(str); + r.f = c10::stod(str); } catch (const std::invalid_argument& e) { throw ErrorReport(token.range) << "Number cannot be converted to double"; @@ -216,7 +216,7 @@ ParsedLiteral IRParser::parseScalarLiteral(Node* n) { } else { r.k = AttributeKind::i; try { - r.i = std::stoll(str); + r.i = c10::stoll(str); } catch (const std::invalid_argument& e) { throw ErrorReport(token.range) << "Number cannot be converted to integer"; diff --git a/torch/csrc/jit/jit_opt_limit.cpp b/torch/csrc/jit/jit_opt_limit.cpp index 7ebfb527380..527f64c2580 100644 --- a/torch/csrc/jit/jit_opt_limit.cpp +++ b/torch/csrc/jit/jit_opt_limit.cpp @@ -21,7 +21,7 @@ static std::unordered_map& passes_to_current_counter() { static int parseOptLimit(const std::string& opt_limit) { try { - int64_t n = std::stoi(opt_limit); + int64_t n = c10::stoi(opt_limit); return n; } catch (...) { return -1; diff --git a/torch/csrc/jit/mobile/nnc/aot_compiler.cpp b/torch/csrc/jit/mobile/nnc/aot_compiler.cpp index 3b3fb8af618..7f881978014 100644 --- a/torch/csrc/jit/mobile/nnc/aot_compiler.cpp +++ b/torch/csrc/jit/mobile/nnc/aot_compiler.cpp @@ -250,7 +250,7 @@ static std::vector> parseInputShapes( std::vector input_dims; input_dims.reserve(input_dims_str.size()); for (const auto& s : input_dims_str) { - input_dims.push_back(std::stoi(s)); + input_dims.push_back(c10::stoi(s)); } inputs.push_back(input_dims); } @@ -301,7 +301,7 @@ static std::vector parseInputDynamicShapes( std::vector dynamic_dims; dynamic_dims.reserve(dynamic_dims_list.size()); for (const auto& dim : dynamic_dims_list) { - dynamic_dims.push_back(std::stoi(dim)); + dynamic_dims.push_back(c10::stoi(dim)); } return dynamic_dims; } diff --git a/torch/csrc/jit/runtime/register_prim_ops.cpp b/torch/csrc/jit/runtime/register_prim_ops.cpp index 6ede3a4496e..e626d5ea9c6 100644 --- a/torch/csrc/jit/runtime/register_prim_ops.cpp +++ b/torch/csrc/jit/runtime/register_prim_ops.cpp @@ -292,7 +292,7 @@ static const std::vector opGenArgs{ auto s = pop(stack).toString(); // NOLINTNEXTLINE(cppcoreguidelines-init-variables) std::string::size_type sz; - int64_t val = static_cast(std::stoll(s->string(), &sz)); + int64_t val = static_cast(c10::stoll(s->string(), &sz)); if (sz == s->string().size()) { push(stack, val); } else { @@ -349,7 +349,7 @@ static const std::vector opGenArgs{ auto s = pop(stack).toString(); // NOLINTNEXTLINE(cppcoreguidelines-init-variables) std::string::size_type sz; - double b = std::stod(s->string(), &sz); + double b = c10::stod(s->string(), &sz); if (sz == s->string().size()) { push(stack, b); } else { diff --git a/torch/csrc/jit/testing/file_check.cpp b/torch/csrc/jit/testing/file_check.cpp index b709abc9f4e..a53b98b07d4 100644 --- a/torch/csrc/jit/testing/file_check.cpp +++ b/torch/csrc/jit/testing/file_check.cpp @@ -233,7 +233,7 @@ struct FileCheckImpl { auto count_view = source->text_str() .substr(end_check_string, end - end_check_string) .str(); - count = std::stoll(std::string(count_view.begin(), count_view.end())); + count = c10::stoll(std::string(count_view.begin(), count_view.end())); end_check_string = end + 2; // add ':' and the space } auto check = Check(