diff --git a/tools/python/ort_test_dir_utils.py b/tools/python/ort_test_dir_utils.py
index 890fa1cb46..c3fd9bd683 100644
--- a/tools/python/ort_test_dir_utils.py
+++ b/tools/python/ort_test_dir_utils.py
@@ -82,10 +82,18 @@ def create_test_dir(model_path, root_path, test_name,
     model_path = os.path.abspath(model_path)
     root_path = os.path.abspath(root_path)
     test_dir = os.path.join(root_path, test_name)
-    test_data_dir = os.path.join(test_dir, "test_data_set_0")
+    if not os.path.exists(test_dir):
+        os.makedirs(test_dir)
 
-    if not os.path.exists(test_dir) or not os.path.exists(test_data_dir):
-        os.makedirs(test_data_dir)
+    # add to existing test data sets if present
+    test_num = 0
+    while True:
+        test_data_dir = os.path.join(test_dir, "test_data_set_" + str(test_num))
+        if not os.path.exists(test_data_dir):
+            os.mkdir(test_data_dir)
+            break
+
+        test_num += 1
 
     model_filename = os.path.split(model_path)[-1]
     test_model_filename = os.path.join(test_dir, model_filename)