pytorch/torch/csrc/distributed/c10d/PrefixStore.cpp

83 lines
2.1 KiB
C++
Raw Normal View History

Canonicalize all includes in PyTorch. (#14849) Summary: Anywhere we used #include "foo.h", we now say #include <foo.h> Paths are adjusted to be rooted out of aten/src, torch/lib, or the root level directory. I modified CMakeLists.txt by hand to remove TH and THC from the include paths. I used the following script to do the canonicalization: ``` import subprocess import re import os.path files = subprocess.check_output(['git', 'ls-files']).decode('utf-8').rstrip().split('\n') for fn in files: if not any(fn.endswith(suff) for suff in ['.cu', '.cpp', '.in', '.h', '.hpp', '.cu', '.cuh', '.cc']): continue if not any(fn.startswith(pref) for pref in ["aten/", "torch/"]): continue with open(fn, 'r') as f: c = f.read() def fmt(p): return "#include <{}>".format(p) def repl(m): p = m.group(1) if p in ["dlfcn.h", "unistd.h", "nvrtc.h", "cuda.h", "cuda_runtime.h", "cstdint", "cudnn.h", "Python.h", "cusparse.h", "cuda_runtime_api.h", "cuda_fp16.h", "cublas_v2.h", "stdint.h", "curand_kernel.h"]: return fmt(p) if any(p.startswith(pref) for pref in ["torch/csrc", "c10/", "ATen/", "caffe2/", "TH/", "THC/", "Eigen/", "gtest/", "zdl/", "gloo/", "onnx/", "miopen/"]): return fmt(p) for root in ["aten/src", "torch/lib", ""]: for bad_root in [os.path.dirname(fn), "aten/src/TH", "aten/src/THC", "torch/csrc"]: new_p = os.path.relpath(os.path.join(bad_root, p), root) if not new_p.startswith("../") and (os.path.exists(os.path.join(root, new_p)) or os.path.exists(os.path.join(root, new_p + ".in"))): return fmt(new_p) print("ERROR: ", fn, p) return m.group(0) new_c = re.sub(r'#include "([^"]+)"', repl, c) if new_c != c: print(fn) with open(fn, 'w') as f: f.write(new_c) ``` Signed-off-by: Edward Z. Yang <ezyang@fb.com> Pull Request resolved: https://github.com/pytorch/pytorch/pull/14849 Reviewed By: dzhulgakov Differential Revision: D13363445 Pulled By: ezyang fbshipit-source-id: 52361f878a672785f9306c9e9ab2513128092b68
2018-12-09 03:32:01 +00:00
#include <c10d/PrefixStore.hpp>
namespace c10d {
PrefixStore::PrefixStore(
const std::string& prefix,
c10::intrusive_ptr<Store> store)
: prefix_(prefix), store_(store) {}
std::string PrefixStore::joinKey(const std::string& key) {
return prefix_ + "/" + key;
}
std::vector<std::string> PrefixStore::joinKeys(
const std::vector<std::string>& keys) {
std::vector<std::string> joinedKeys;
joinedKeys.reserve(keys.size());
for (const auto& key : keys) {
joinedKeys.emplace_back(joinKey(key));
}
return joinedKeys;
}
void PrefixStore::set(
const std::string& key,
const std::vector<uint8_t>& value) {
store_->set(joinKey(key), value);
}
std::vector<uint8_t> PrefixStore::compareSet(
const std::string& key,
const std::vector<uint8_t>& expectedValue,
const std::vector<uint8_t>& desiredValue) {
return store_->compareSet(joinKey(key), expectedValue, desiredValue);
}
std::vector<uint8_t> PrefixStore::get(const std::string& key) {
return store_->get(joinKey(key));
}
int64_t PrefixStore::add(const std::string& key, int64_t value) {
return store_->add(joinKey(key), value);
}
bool PrefixStore::deleteKey(const std::string& key) {
return store_->deleteKey(joinKey(key));
}
void PrefixStore::watchKey(const std::string& key, WatchKeyCallback callback) {
return store_->watchKey(joinKey(key), std::move(callback));
}
int64_t PrefixStore::getNumKeys() {
return store_->getNumKeys();
}
bool PrefixStore::check(const std::vector<std::string>& keys) {
auto joinedKeys = joinKeys(keys);
return store_->check(joinedKeys);
}
void PrefixStore::wait(const std::vector<std::string>& keys) {
auto joinedKeys = joinKeys(keys);
store_->wait(joinedKeys);
}
void PrefixStore::wait(
const std::vector<std::string>& keys,
const std::chrono::milliseconds& timeout) {
auto joinedKeys = joinKeys(keys);
store_->wait(joinedKeys, timeout);
}
const std::chrono::milliseconds& PrefixStore::getTimeout() const noexcept {
return store_->getTimeout();
}
void PrefixStore::setTimeout(const std::chrono::milliseconds& timeout) {
store_->setTimeout(timeout);
}
} // namespace c10d