diff --git a/.gitignore b/.gitignore index be2375d256..d27dedbc2b 100644 --- a/.gitignore +++ b/.gitignore @@ -49,10 +49,6 @@ java/gradle java/.gradle java/hs_*.log onnxruntime/python/version_info.py -/tools/perf_util/target/classes/com/msft/send_perf_metrics -/tools/perf_util/send_perf_metrics.iml -/tools/perf_util/target/classes -/tools/perf_util/src/main/resources /orttraining/orttraining/eager/ort_aten.g.cpp /orttraining/orttraining/eager/ort_customops.g.cpp /csharp/**/packages diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-perf-test-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-perf-test-ci-pipeline.yml deleted file mode 100644 index bb56fb7c34..0000000000 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-perf-test-ci-pipeline.yml +++ /dev/null @@ -1,90 +0,0 @@ -trigger: none - -jobs: -- job: Onnxruntime_Linux_GPU_Training_Perf_Test - - timeoutInMinutes: 120 - - variables: - - group: 'ortperf' # variable group - - steps: - - checkout: self - clean: true - submodules: recursive - - - template: templates/run-docker-build-steps.yml - parameters: - RunDockerBuildArgs: > - -o ubuntu20.04 -d gpu - -t onnxruntime_perf_test_image - -x " - --config RelWithDebInfo - --enable_training - --update --build --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=70 - " - DisplayName: 'Build performance tests' - - - bash: tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_test_data.sh -p $(orttrainingtestdatascus-storage-key) -s "//orttrainingtestdatascus.file.core.windows.net/bert-data" -d "/bert_data" - displayName: 'Mount bert-data' - condition: succeededOrFailed() # ensure all tests are run - - - bash: tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_test_data.sh -p $(orttrainingtestdatascus-storage-key) -s "//orttrainingtestdatascus.file.core.windows.net/gpt2-data" -d "/gpt2_data" - displayName: 'Mount gpt2 test data' - condition: succeededOrFailed() # ensure all tests are run - - - script: > - docker run --gpus all --rm --name onnxruntime-gpu-perf - --volume $(Build.SourcesDirectory):/onnxruntime_src - --volume $(Build.BinariesDirectory):/build - --volume /bert_data/bert_models:/build/bert_models:ro - --volume /bert_data:/build/bert_data:ro - -e NIGHTLY_BUILD onnxruntime_perf_test_image - /usr/bin/python3 /onnxruntime_src/orttraining/tools/ci_test/run_bert_perf_test.py - --binary_dir /build/RelWithDebInfo - --training_data_root /build/bert_data - --model_root /build/bert_models - displayName: 'Run bert performance tests' - condition: succeededOrFailed() - timeoutInMinutes: 120 - - - script: > - docker run --gpus all --rm --name onnxruntime-gpu-perf - --volume $(Build.SourcesDirectory):/onnxruntime_src - --volume $(Build.BinariesDirectory):/build - --volume /gpt2_data/gpt2_models:/build/gpt2_models:ro - --volume /gpt2_data:/build/gpt2_data:ro - -e NIGHTLY_BUILD onnxruntime_perf_test_image - /usr/bin/python3 /onnxruntime_src/orttraining/tools/ci_test/run_gpt2_perf_test.py - --binary_dir /build/RelWithDebInfo - --training_data_root /build/gpt2_data - --model_root /build/gpt2_models - displayName: 'Run gpt-2 performance tests' - condition: succeededOrFailed() - timeoutInMinutes: 120 - - # generate jdbc.properties - - script: > - mkdir -p $(Build.SourcesDirectory)/tools/perf_util/src/main/resources && - printf "url=jdbc:mysql://onnxruntimedashboard.mysql.database.azure.com/onnxruntime?serverTimezone=UTC&useUnicode=true&characterEncoding=UTF-8\nuser=powerbi@onnxruntimedashboard\npassword_env=ORT_PERF_PASSWORD" - > $(Build.SourcesDirectory)/tools/perf_util/src/main/resources/jdbc.properties - displayName: 'Create resource file' - - - script: > - mvn package - displayName: 'Maven build' - workingDirectory: $(Build.SourcesDirectory)/tools/perf_util - - # process json files - - script: > - java -cp target/send_perf_metrics-0.0.1-SNAPSHOT-jar-with-dependencies.jar com.msft.send_perf_metrics.App "$(Build.SourcesDirectory)/orttraining/tools/ci_test/results" - env: - ORT_PERF_PASSWORD: $(ortperf) - displayName: 'Populate perf metrics' - workingDirectory: $(Build.SourcesDirectory)/tools/perf_util - - - template: templates/component-governance-component-detection-steps.yml - parameters: - condition: 'succeeded' - - - template: templates/clean-agent-build-directory-step.yml diff --git a/tools/perf_util/pom.xml b/tools/perf_util/pom.xml deleted file mode 100644 index b5e129f4af..0000000000 --- a/tools/perf_util/pom.xml +++ /dev/null @@ -1,56 +0,0 @@ - - 4.0.0 - - com.msft - send_perf_metrics - 0.0.1-SNAPSHOT - jar - - send_perf_metrics - http://maven.apache.org - - - - - maven-assembly-plugin - 3.1.1 - - - jar-with-dependencies - - - - - make-assembly - package - - single - - - - - - - - UTF-8 - 1.8 - 1.8 - - - - - - com.googlecode.json-simple - json-simple - 1.1.1 - - - - mysql - mysql-connector-java - 8.0.22 - - - diff --git a/tools/perf_util/src/main/java/com/msft/send_perf_metrics/App.java b/tools/perf_util/src/main/java/com/msft/send_perf_metrics/App.java deleted file mode 100644 index a0a04211f6..0000000000 --- a/tools/perf_util/src/main/java/com/msft/send_perf_metrics/App.java +++ /dev/null @@ -1,168 +0,0 @@ -package com.msft.send_perf_metrics; - -import org.json.simple.JSONObject; -import org.json.simple.parser.JSONParser; - -import java.io.*; -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; - -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.text.SimpleDateFormat; -import java.util.*; - -public class App { - - static String exec_command(Path source_dir, String... commands) throws Exception { - ProcessBuilder sb = new ProcessBuilder(commands).directory(source_dir.toFile()).redirectErrorStream(true); - Process p = sb.start(); - if (p.waitFor() != 0) - throw new RuntimeException("execute " + String.join(" ", commands) + " failed"); - try (BufferedReader r = new BufferedReader(new InputStreamReader(p.getInputStream()))) { - return r.readLine(); - } - } - - public static void main(String[] args) throws Exception { - - final Path source_dir = Paths.get(args[0]); - final List perf_metrics = new ArrayList(); - Files.walkFileTree(source_dir, new SimpleFileVisitor() { - - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { - String dirname = dir.getFileName().toString(); - if (dirname != "." && dirname.startsWith(".")) - return FileVisitResult.SKIP_SUBTREE; - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - String filename = file.getFileName().toString(); - - if (!filename.startsWith(".") && filename.endsWith(".json")) { - perf_metrics.add(file); - System.out.println(filename); - } - return FileVisitResult.CONTINUE; - } - - }); - - final Path cwd_dir = Paths.get(System.getProperty("user.dir")); - // git rev-parse HEAD - String commit_id = exec_command(cwd_dir, "git", "rev-parse", "HEAD"); - String date = exec_command(cwd_dir, "git", "show", "-s", "--format=%ci", commit_id); - final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss Z"); - java.util.Date commitDate = sdf.parse(date); - final SimpleDateFormat simple_date_format = new SimpleDateFormat("yyyy-MM-dd"); - String batch_id = simple_date_format.format(commitDate); - System.out.println(String.format("Commit change date: %s", batch_id)); - - // collect all json files list - processPerfMetrics(perf_metrics, commit_id, batch_id); - - // TODO - add e2e tests later, run it w/ process command - } - - private static void processPerfMetrics(final List perf_metrics, String commit_id, - String batch_id) throws Exception { - try { - Connection conn = JdbcUtil.GetConn(); - System.out.println("MySQL DB connection established.\n"); - // go thru each json file - JSONParser jsonParser = new JSONParser(); - for (Path metrics_json : perf_metrics) { - try (FileReader reader = new FileReader(metrics_json.toAbsolutePath().toString())) { - // Read JSON file - Object obj = jsonParser.parse(reader); - loadMetricsIntoMySQL(conn, commit_id, batch_id, (JSONObject) obj); - } - } - } catch (Exception e) { - e.printStackTrace(); - throw e; - } - } - - - static private void loadMetricsIntoMySQL(java.sql.Connection conn, String commit_id, String batch_id, - JSONObject json_object) throws Exception { - - // field name -> json value - Map field_mapping = new LinkedHashMap(); - Set update_on_duplicate_fields = - new LinkedHashSet<> (Arrays.asList("AvgTimePerBatch", "Throughput", "StabilizedThroughput", "EndToEndThroughput", "TotalTime", "AvgCPU", "Memory")); - - field_mapping.put("BatchId", batch_id); - field_mapping.put("CommitId", commit_id.substring(0, 8)); - json_object.forEach((key, value) -> { - if (key.equals("DerivedProperties")) { - JSONObject properties = (JSONObject) json_object.get("DerivedProperties"); - properties.forEach((sub_key, sub_value) -> { - field_mapping.put((String)sub_key, sub_value); - }); - } else { - field_mapping.put((String)key, value); - } - }); - - // building sql statement - StringBuilder sb = new StringBuilder("INSERT INTO perf_test_training_data ("); - field_mapping.forEach((key, value) -> { - sb.append(key).append(","); - }); - sb.append("Time) values ("); - for(int i = 0; i < field_mapping.size(); i++) { - sb.append("?,"); - } - sb.append("Now()) ON DUPLICATE KEY UPDATE "); - update_on_duplicate_fields.forEach((key) -> { - if(field_mapping.get(key) != null) { - sb.append(key).append("=?,"); - } - }); - - try (java.sql.PreparedStatement st = conn.prepareStatement(sb.substring(0, sb.length() - 1))) { - int i = 0; // param index - for (Map.Entry entry : field_mapping.entrySet()) { - setSqlParam(++i, st, entry.getValue()); - } - - // update section - for(String key : update_on_duplicate_fields) { - Object value = field_mapping.get(key); - if(value != null) { - setSqlParam(++i, st, value); - } - } - - st.executeUpdate(); - } catch (Exception e) { - e.printStackTrace(); - throw e; - } - - } - - static void setSqlParam(int param_index, PreparedStatement st, Object value) throws Exception { - if (value instanceof String) { - st.setString(param_index, (String) value); - } else if (value instanceof Long) { - st.setInt(param_index, (int) (long) value); - } else if (value instanceof Double) { - st.setFloat(param_index, (float) (double) value); - } else if (value instanceof Boolean) { - st.setBoolean(param_index, (Boolean) value); - } else { - throw new Exception("Unsupported data type:" + value.getClass().getName()); - } - } - -} diff --git a/tools/perf_util/src/main/java/com/msft/send_perf_metrics/JdbcUtil.java b/tools/perf_util/src/main/java/com/msft/send_perf_metrics/JdbcUtil.java deleted file mode 100644 index d7132f7f26..0000000000 --- a/tools/perf_util/src/main/java/com/msft/send_perf_metrics/JdbcUtil.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.msft.send_perf_metrics; - -import java.sql.DriverManager; -import java.util.Map; -import java.util.Properties; - -public class JdbcUtil { - static java.sql.Connection GetConn() throws Exception { - try (java.io.InputStream in = App.class.getResourceAsStream("/jdbc.properties")) { - if (in == null) - throw new RuntimeException("Error reading jdbc properties"); - Properties props = new Properties(); - props.load(in); - // loading password via env variable - return DriverManager.getConnection(props.getProperty("url"), props.getProperty("user"), - System.getenv(props.getProperty("password_env"))); - } - } -}