diff --git a/onnxruntime/core/framework/tensor_external_data_info.cc b/onnxruntime/core/framework/tensor_external_data_info.cc
index 8bbf30aee0..761d57c1b9 100644
--- a/onnxruntime/core/framework/tensor_external_data_info.cc
+++ b/onnxruntime/core/framework/tensor_external_data_info.cc
@@ -26,12 +26,16 @@ Status ExternalDataInfo::Create(const RepeatedPtrField<StringStringEntryProto>&
       out->rel_path_ = ToWideString(stringmap.value());
     } else if (stringmap.key() == "offset" && !stringmap.value().empty()) {
       char* end;
+#ifdef _WIN32
+      out->offset_ = _strtoi64(stringmap.value().c_str(), &end, 10);
+#else
       out->offset_ = OrtStrToPtrDiff(stringmap.value().c_str(), &end);
+#endif
       if (end != stringmap.value().c_str() + stringmap.value().length())
         return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "parsing ", stringmap.value(), " failed");
     } else if (stringmap.key() == "length" && !stringmap.value().empty()) {
       char* end;
-      out->length_ = OrtStrToPtrDiff(stringmap.value().c_str(), &end);
+      out->length_ = static_cast<size_t>(OrtStrToPtrDiff(stringmap.value().c_str(), &end));
       if (end != stringmap.value().c_str() + stringmap.value().length())
         return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "parsing ", stringmap.value(), " failed");
     } else if (stringmap.key() == "checksum" && !stringmap.value().empty()) {
diff --git a/onnxruntime/core/framework/tensor_external_data_info.h b/onnxruntime/core/framework/tensor_external_data_info.h
index 575e19c6e7..3445d8cd67 100644
--- a/onnxruntime/core/framework/tensor_external_data_info.h
+++ b/onnxruntime/core/framework/tensor_external_data_info.h
@@ -10,16 +10,23 @@
 namespace onnxruntime {
 class ExternalDataInfo {
  private:
+#ifdef _WIN32
+  using OFFSET_TYPE = int64_t;
+#else
+  using OFFSET_TYPE = off_t;
+#endif
   std::basic_string<ORTCHAR_T> rel_path_;
-  ptrdiff_t offset_ = 0;
-  ptrdiff_t length_ = 0;
+  OFFSET_TYPE offset_ = 0;
+
+  // 0 means the whole file
+  size_t length_ = 0;
   std::string checksum_;
 
  public:
   const std::basic_string<ORTCHAR_T>& GetRelPath() const { return rel_path_; }
 
-  ptrdiff_t GetOffset() const { return offset_; }
-  ptrdiff_t GetLength() const { return length_; }
+  OFFSET_TYPE GetOffset() const { return offset_; }
+  size_t GetLength() const { return length_; }
 
   const std::string& GetChecksum() const { return checksum_; }
 
diff --git a/onnxruntime/core/framework/tensorprotoutils.cc b/onnxruntime/core/framework/tensorprotoutils.cc
index 18fe26dde9..3e52be92c6 100644
--- a/onnxruntime/core/framework/tensorprotoutils.cc
+++ b/onnxruntime/core/framework/tensorprotoutils.cc
@@ -381,9 +381,6 @@ Status TensorProtoToMLValue(const Env& env, const ORTCHAR_T* tensor_proto_path,
 
       std::unique_ptr<ExternalDataInfo> external_data_info;
       ORT_RETURN_IF_ERROR(ExternalDataInfo::Create(tensor_proto.external_data(), external_data_info));
-      if (external_data_info->GetOffset() > 0) {
-        return ORT_MAKE_STATUS(ONNXRUNTIME, NOT_IMPLEMENTED, "Cannot support tensor data with offset > 0");
-      }
       std::basic_string<ORTCHAR_T> full_path;
       if (tensor_proto_path != nullptr) {
         ORT_RETURN_IF_ERROR(GetDirNameFromFilePath(tensor_proto_path, full_path));
@@ -391,11 +388,12 @@ Status TensorProtoToMLValue(const Env& env, const ORTCHAR_T* tensor_proto_path,
       } else {
         full_path = external_data_info->GetRelPath();
       }
-
+      raw_data_len = external_data_info->GetLength();
       // load the file
       {
         void* file_data;
-        ORT_RETURN_IF_ERROR(env.ReadFileAsString(full_path.c_str(), file_data, raw_data_len, deleter_for_file_data.d));
+        ORT_RETURN_IF_ERROR(env.ReadFileAsString(full_path.c_str(), external_data_info->GetOffset(),
+            file_data, raw_data_len, deleter_for_file_data.d));
         raw_data = file_data;
       }
     } else if (tensor_proto.has_raw_data()) {
diff --git a/onnxruntime/core/platform/env.h b/onnxruntime/core/platform/env.h
index ba03bc0bce..9578397bfe 100644
--- a/onnxruntime/core/platform/env.h
+++ b/onnxruntime/core/platform/env.h
@@ -102,12 +102,14 @@ class Env {
    *
    * \param file_path file_path must point to a regular file, which can't be a pipe/socket/...
    * \param[out] p  allocated buffer with the file data
-   * \param[out] len lenght of p
+   * \param[in] offset file offset. If offset>0, then len must also be >0.
+   * \param[in, out] len length to read(or has read). If len==0, read the whole file.
    * @return
    */
-  virtual common::Status ReadFileAsString(const char* file_path, void*& p, size_t& len, OrtCallback& deleter) const = 0;
+  virtual common::Status ReadFileAsString(const char* file_path, off_t offset, void*& p, size_t& len,
+      OrtCallback& deleter) const = 0;
 #else
-  virtual common::Status ReadFileAsString(const wchar_t* file_path, void*& p, size_t& len,
+  virtual common::Status ReadFileAsString(const wchar_t* file_path, int64_t offset, void*& p, size_t& len,
                                           OrtCallback& deleter) const = 0;
 #endif
 
diff --git a/onnxruntime/core/platform/posix/env.cc b/onnxruntime/core/platform/posix/env.cc
index 21d4d97bf4..78ff664e1e 100644
--- a/onnxruntime/core/platform/posix/env.cc
+++ b/onnxruntime/core/platform/posix/env.cc
@@ -31,6 +31,11 @@ limitations under the License.
 #include "core/common/common.h"
 #include "core/common/logging/logging.h"
 
+// MAC OS X doesn't have this macro
+#ifndef TEMP_FAILURE_RETRY
+#define TEMP_FAILURE_RETRY(X) X
+#endif
+
 namespace onnxruntime {
 
 namespace {
@@ -116,34 +121,50 @@ class PosixEnv : public Env {
     return getpid();
   }
 
-  static common::Status ReadBinaryFile(int fd, const char* fname, const struct stat& stbuf, void*& p, size_t& len,
+  static common::Status ReadBinaryFile(int fd, off_t offset, const char* fname, void*& p, size_t len,
                                        OrtCallback& deleter) {
-    std::unique_ptr<char[]> buffer(reinterpret_cast<char*>(malloc(stbuf.st_size)));
+    std::unique_ptr<char[]> buffer(reinterpret_cast<char*>(malloc(len)));
     char* wptr = reinterpret_cast<char*>(buffer.get());
-    auto length_remain = stbuf.st_size;
+    auto length_remain = len;
     do {
       size_t bytes_to_read = length_remain;
-      ssize_t bytes_readed = read(fd, wptr, bytes_to_read);
-      if (bytes_readed <= 0) {
+      ssize_t bytes_read;
+      TEMP_FAILURE_RETRY(bytes_read =
+                             offset > 0 ? pread(fd, wptr, bytes_to_read, offset) : read(fd, wptr, bytes_to_read));
+      if (bytes_read <= 0) {
         int err = errno;
-        (void)close(fd);
         return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "read file '", fname, "' fail, error code = ", err);
       }
-      assert(static_cast<size_t>(bytes_readed) <= bytes_to_read);
-      wptr += bytes_readed;
-      length_remain -= bytes_readed;
+      assert(static_cast<size_t>(bytes_read) <= bytes_to_read);
+      wptr += bytes_read;
+      length_remain -= bytes_read;
     } while (length_remain > 0);
     p = buffer.release();
-    len = stbuf.st_size;
     deleter.f = DeleteBuffer;
     deleter.param = p;
     return Status::OK();
   }
 
-  common::Status ReadFileAsString(const char* fname, void*& p, size_t& len, OrtCallback& deleter) const override {
+  static bool GetFileSizeIfUnknown(int fd, size_t& len) {
+    if(len > 0) return true;
+    struct stat stbuf;
+    if ((fstat(fd, &stbuf) != 0) || (!S_ISREG(stbuf.st_mode))) {
+      return false;
+    }
+    len = static_cast<size_t>(stbuf.st_size);
+    return true;
+  }
+
+  common::Status ReadFileAsString(const char* fname, off_t offset, void*& p, size_t& len,
+      OrtCallback& deleter) const override {
     if (!fname) {
       return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "ReadFileAsString: 'fname' cannot be NULL");
     }
+
+    if (offset < 0) {
+      return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT,
+                            "ReadFileAsString: offset must be non-negative");
+    }
     deleter.f = nullptr;
     deleter.param = nullptr;
     int fd = open(fname, O_RDONLY);
@@ -151,35 +172,27 @@ class PosixEnv : public Env {
       int err = errno;
       return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "open file ", fname, " fail, errcode =", err);
     }
-    struct stat stbuf;
-    if ((fstat(fd, &stbuf) != 0) || (!S_ISREG(stbuf.st_mode))) {
+    if (!GetFileSizeIfUnknown(fd, len)) {
       (void)close(fd);
       return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Get file '", fname, "' size fail");
     }
-    if (stbuf.st_size == 0) {
+    if (len == 0) {
       p = nullptr;
-      len = 0;
     } else {
-      if (sizeof(fname) <= 4) {
-        auto st = ReadBinaryFile(fd, fname, stbuf, p, len, deleter);
+      long page_size = sysconf(_SC_PAGESIZE);
+      off_t offset_to_page = offset % static_cast<off_t>(page_size);
+      p = mmap(nullptr, len + offset_to_page, PROT_READ, MAP_SHARED, fd, offset - offset_to_page);
+      if (p == MAP_FAILED) {
+        auto st = ReadBinaryFile(fd, offset, fname, p, len, deleter);
         (void)close(fd);
         if (!st.IsOK()) {
           return st;
         }
       } else {
-        size_t flen = static_cast<size_t>(stbuf.st_size);
-        p = mmap(NULL, flen, PROT_READ, MAP_SHARED, fd, 0);
-        if (p == MAP_FAILED) {
-          auto st = ReadBinaryFile(fd, fname, stbuf, p, len, deleter);
-          (void)close(fd);
-          if (!st.IsOK()) {
-            return st;
-          }
-        } else {
-          len = stbuf.st_size;
-          deleter.f = UnmapFile;
-          deleter.param = new UnmapFileParam{p, flen, fd};
-        }
+        // leave the file open
+        deleter.f = UnmapFile;
+        deleter.param = new UnmapFileParam{p, len + offset_to_page, fd};
+        p = reinterpret_cast<char*>(p) + offset_to_page;
       }
     }
 
diff --git a/onnxruntime/core/platform/windows/env.cc b/onnxruntime/core/platform/windows/env.cc
index 8f1cf920e7..84dafd94c5 100644
--- a/onnxruntime/core/platform/windows/env.cc
+++ b/onnxruntime/core/platform/windows/env.cc
@@ -94,10 +94,30 @@ class WindowsEnv : public Env {
   void ExecuteTask(const Task& t) const override {
     t.f();
   }
-  common::Status ReadFileAsString(const wchar_t* fname, void*& p, size_t& len, OrtCallback& deleter) const override {
+
+  static common::Status GetFileSizeIfUnknown(const wchar_t* fname, HANDLE hFile, size_t& length) {
+    if (length > 0) return Status::OK();
+    LARGE_INTEGER filesize;
+    if (!GetFileSizeEx(hFile, &filesize)) {
+      int err = GetLastError();
+      return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "GetFileSizeEx ", ToMBString(fname), " fail, errcode =", err);
+    }
+    if (static_cast<ULONGLONG>(filesize.QuadPart) > std::numeric_limits<size_t>::max()) {
+      return common::Status(common::ONNXRUNTIME, common::FAIL, "ReadFileAsString: File is too large");
+    }
+    length = static_cast<size_t>(filesize.QuadPart);
+    return Status::OK();
+  }
+
+  common::Status ReadFileAsString(const wchar_t* fname, int64_t offset, void*& p, size_t& len,
+                                  OrtCallback& deleter) const override {
     if (!fname) {
       return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "ReadFileAsString: 'fname' cannot be NULL");
     }
+    if (offset > 0 && len == 0) {
+      return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT,
+                            "ReadFileAsString: please specify length to read");
+    }
     deleter.f = nullptr;
     deleter.param = nullptr;
     HANDLE hFile = CreateFileW(fname, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
@@ -106,22 +126,26 @@ class WindowsEnv : public Env {
       return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "open file ", ToMBString(fname), " fail, errcode =", err);
     }
     std::unique_ptr<void, decltype(&CloseHandle)> handler_holder(hFile, CloseHandle);
-    LARGE_INTEGER filesize;
-    if (!GetFileSizeEx(hFile, &filesize)) {
-      int err = GetLastError();
-      return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "GetFileSizeEx ", ToMBString(fname), " fail, errcode =", err);
-    }
+    ORT_RETURN_IF_ERROR(GetFileSizeIfUnknown(fname, hFile, len));
     // check the file file for avoiding allocating a zero length buffer
-    if (filesize.QuadPart == 0) {  // empty file
+    if (len == 0) {  // empty file
       p = nullptr;
       len = 0;
       return Status::OK();
     }
-    std::unique_ptr<char[]> buffer(reinterpret_cast<char*>(malloc(filesize.QuadPart)));
+    std::unique_ptr<char[]> buffer(reinterpret_cast<char*>(malloc(len)));
     char* wptr = reinterpret_cast<char*>(buffer.get());
-    auto length_remain = filesize.QuadPart;
-    DWORD readed = 0;
-    for (; length_remain > 0; wptr += readed, length_remain -= readed) {
+    size_t length_remain = len;
+    DWORD bytes_read = 0;
+    if (offset > 0) {
+      LARGE_INTEGER liCurrentPosition;
+      liCurrentPosition.QuadPart = offset;
+      if (SetFilePointerEx(hFile, liCurrentPosition, &liCurrentPosition, FILE_BEGIN) != TRUE) {
+        int err = GetLastError();
+        return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "SetFilePointerEx ", ToMBString(fname), " fail, errcode =", err);
+      }
+    }
+    for (; length_remain > 0; wptr += bytes_read, length_remain -= bytes_read) {
       //read at most 1GB each time
       DWORD bytes_to_read;
       if (length_remain > (1 << 30)) {
@@ -129,20 +153,19 @@ class WindowsEnv : public Env {
       } else {
         bytes_to_read = static_cast<DWORD>(length_remain);
       }
-      if (ReadFile(hFile, wptr, bytes_to_read, &readed, nullptr) != TRUE) {
+      if (ReadFile(hFile, wptr, bytes_to_read, &bytes_read, nullptr) != TRUE) {
         int err = GetLastError();
         p = nullptr;
         len = 0;
         return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "ReadFile ", ToMBString(fname), " fail, errcode =", err);
       }
-      if (readed != bytes_to_read) {
+      if (bytes_read != bytes_to_read) {
         p = nullptr;
         len = 0;
         return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "ReadFile ", ToMBString(fname), " fail: unexpected end");
       }
     }
     p = buffer.release();
-    len = filesize.QuadPart;
     deleter.f = DeleteBuffer;
     deleter.param = p;
     return common::Status::OK();
diff --git a/onnxruntime/core/providers/cpu/symbols.txt b/onnxruntime/core/providers/cpu/symbols.txt
index 127ae0b866..f79af6fc79 100644
--- a/onnxruntime/core/providers/cpu/symbols.txt
+++ b/onnxruntime/core/providers/cpu/symbols.txt
@@ -62,6 +62,7 @@ OrtReleaseTensorTypeAndShapeInfo
 OrtReleaseTypeInfo
 OrtReleaseValue
 OrtRun
+OrtRunCallback
 OrtRunOptionsGetRunLogVerbosityLevel
 OrtRunOptionsGetRunTag
 OrtRunOptionsSetRunLogVerbosityLevel
diff --git a/onnxruntime/test/onnx/gen_test_models.py b/onnxruntime/test/onnx/gen_test_models.py
index f999ef5ed2..afdbf5507c 100644
--- a/onnxruntime/test/onnx/gen_test_models.py
+++ b/onnxruntime/test/onnx/gen_test_models.py
@@ -40,6 +40,8 @@ def generate_abs_op_test(type, X, top_test_folder):
         model_def = helper.make_model(graph_def, producer_name='onnx-example')
         #final_model = onnx.utils.polish_model(model_def)
         final_model = model_def
+        if is_raw:
+            onnx.external_data_helper.convert_model_to_external_data(final_model, True)
         onnx.save(final_model, os.path.join(test_folder, 'model.onnx'))
         expected_output_array = np.abs(X)
         expected_output_tensor = numpy_helper.from_array(expected_output_array)
@@ -85,7 +87,6 @@ def test_size(output_dir):
     generate_size_op_test(TensorProto.FLOAT, np.random.randn(100, 3000, 10).astype(np.float32), os.path.join(output_dir,'test_size_float'))
     generate_size_op_test(TensorProto.STRING, np.array(['abc', 'xy'], dtype=np.bytes_), os.path.join(output_dir,'test_size_string'))
 
-np.array(['abc', 'xy'])
 args = parse_arguments()
 os.makedirs(args.output_dir,exist_ok=True)
 test_abs(args.output_dir)
diff --git a/onnxruntime/test/shared_lib/test_tensor_loader.cc b/onnxruntime/test/shared_lib/test_tensor_loader.cc
index 4f674ec312..b3c1e28960 100644
--- a/onnxruntime/test/shared_lib/test_tensor_loader.cc
+++ b/onnxruntime/test/shared_lib/test_tensor_loader.cc
@@ -61,6 +61,7 @@ TEST_F(CApiTest, load_simple_float_tensor) {
   ASSERT_EQ(real_output[1], 2.2f);
   ASSERT_EQ(real_output[2], 3.5f);
   OrtReleaseValue(value);
+  OrtRunCallback(deleter);
 }
 
 template <bool use_current_dir>
@@ -116,6 +117,7 @@ static void run_external_data_test() {
   ASSERT_EQ(real_output[1], 2.2f);
   ASSERT_EQ(real_output[2], 3.5f);
   OrtReleaseValue(value);
+  OrtRunCallback(deleter);
 }
 TEST_F(CApiTest, load_float_tensor_with_external_data) {
   run_external_data_test<true>();
@@ -164,6 +166,7 @@ TEST_F(CApiTest, load_huge_tensor_with_external_data) {
     ASSERT_EQ(1, buffer[i]);
   }
   OrtReleaseValue(value);
+  OrtRunCallback(deleter);
 }
 #endif
 }  // namespace test