[Inductor] [CI] [CUDA] Skip the failed models and tests the better way (#127150)

Address subtasks in https://github.com/pytorch/pytorch/issues/126692

After enabling the disabled shards, the following two models regressed (for cu124 configuration):
dynamic_inductor_timm_training.csv
cspdarknet53,pass,7   (expected)                                        | cspdarknet53,fail_accuracy,7           (actual)
eca_botnext26ts_256,pass,7        (expected)                            | eca_botnext26ts_256,fail_accuracy,7 (actual)

Pull Request resolved: https://github.com/pytorch/pytorch/pull/127150
Approved by: https://github.com/huydhn, https://github.com/eqy, https://github.com/atalman

.ci/pytorch/test.sh

@@ -264,6 +264,18 @@ elif [[ $TEST_CONFIG == 'nogpu_AVX512' ]]; then
   export ATEN_CPU_CAPABILITY=avx2
 fi
 
+# temp workarounds for https://github.com/pytorch/pytorch/issues/126692, remove when fixed
+if [[ "$BUILD_ENVIRONMENT" != *-bazel-* ]]; then
+  pushd test
+  CUDA_VERSION=$(python -c "import torch; print(torch.version.cuda)")
+  if [ "$CUDA_VERSION" == "12.4" ]; then
+    ISCUDA124="cu124"
+  else
+    ISCUDA124=""
+  fi
+  popd
+fi
+
 test_python_legacy_jit() {
   time python test/run_test.py --include test_jit_legacy test_jit_fuser_legacy --verbose
   assert_git_not_dirty
@@ -364,7 +376,7 @@ test_inductor_cpp_wrapper_abi_compatible() {
     --output "$TEST_REPORTS_DIR/inductor_cpp_wrapper_training.csv"
   python benchmarks/dynamo/check_accuracy.py \
     --actual "$TEST_REPORTS_DIR/inductor_cpp_wrapper_training.csv" \
-    --expected "benchmarks/dynamo/ci_expected_accuracy/inductor_timm_training.csv"
+    --expected "benchmarks/dynamo/ci_expected_accuracy/${ISCUDA124}/inductor_timm_training.csv"
 }
 
 # "Global" flags for inductor benchmarking controlled by TEST_CONFIG
@@ -526,10 +538,10 @@ test_single_dynamo_benchmark() {
       --output "$TEST_REPORTS_DIR/${name}_${suite}.csv"
     python benchmarks/dynamo/check_accuracy.py \
       --actual "$TEST_REPORTS_DIR/${name}_$suite.csv" \
-      --expected "benchmarks/dynamo/ci_expected_accuracy/${TEST_CONFIG}_${name}.csv"
+      --expected "benchmarks/dynamo/ci_expected_accuracy/${ISCUDA124}/${TEST_CONFIG}_${name}.csv"
     python benchmarks/dynamo/check_graph_breaks.py \
       --actual "$TEST_REPORTS_DIR/${name}_$suite.csv" \
-      --expected "benchmarks/dynamo/ci_expected_accuracy/${TEST_CONFIG}_${name}.csv"
+      --expected "benchmarks/dynamo/ci_expected_accuracy/${ISCUDA124}/${TEST_CONFIG}_${name}.csv"
   fi
 }
 
@@ -576,7 +588,7 @@ test_inductor_torchbench_smoketest_perf() {
     --bfloat16 --inference --inductor --only moco --output "$TEST_REPORTS_DIR/inductor_cpp_wrapper_inference.csv"
   python benchmarks/dynamo/check_accuracy.py \
     --actual "$TEST_REPORTS_DIR/inductor_cpp_wrapper_inference.csv" \
-    --expected "benchmarks/dynamo/ci_expected_accuracy/inductor_torchbench_inference.csv"
+    --expected "benchmarks/dynamo/ci_expected_accuracy/${ISCUDA124}/inductor_torchbench_inference.csv"
 
   python benchmarks/dynamo/torchbench.py --device cuda --performance --backend inductor --float16 --training \
     --batch-size-file "$(realpath benchmarks/dynamo/torchbench_models_list.txt)" --only hf_Bert \
@@ -591,7 +603,13 @@ test_inductor_torchbench_smoketest_perf() {
   # https://github.com/pytorch/pytorch/actions/runs/7158691360/job/19491437314,
   # and thus we lower its threshold to reduce flakiness. If this continues to be a problem,
   # we switch to use some other model.
-  python benchmarks/dynamo/check_perf_csv.py -f "$TEST_REPORTS_DIR/inductor_inference_smoketest.csv" -t 4.9
+  # Use 4.7 for cuda 12.4, change back to 4.9 after fixing https://github.com/pytorch/pytorch/issues/126692
+  if [ "$CUDA_VERSION" == "12.4" ]; then
+    THRESHOLD=4.7
+  else
+    THRESHOLD=4.9
+  fi
+  python benchmarks/dynamo/check_perf_csv.py -f "$TEST_REPORTS_DIR/inductor_inference_smoketest.csv" -t $THRESHOLD
 
   # Check memory compression ratio for a few models
   for test in hf_Albert timm_vision_transformer; do
@@ -610,7 +628,7 @@ test_inductor_torchbench_smoketest_perf() {
       --only $test --output "$TEST_REPORTS_DIR/inductor_warm_start_smoketest_$test.csv"
     python benchmarks/dynamo/check_accuracy.py \
       --actual "$TEST_REPORTS_DIR/inductor_warm_start_smoketest_$test.csv" \
-      --expected "benchmarks/dynamo/ci_expected_accuracy/inductor_huggingface_training.csv"
+      --expected "benchmarks/dynamo/ci_expected_accuracy/${ISCUDA124}/inductor_huggingface_training.csv"
   done
 }
 

.github/workflows/inductor.yml

@@ -140,11 +140,15 @@ jobs:
           { config: "inductor", shard: 1, num_shards: 1, runner: "linux.g5.4xlarge.nvidia.gpu" },
           { config: "inductor_distributed", shard: 1, num_shards: 1, runner: "linux.g5.12xlarge.nvidia.gpu" },
           { config: "inductor_huggingface", shard: 1, num_shards: 1, runner: "linux.g5.4xlarge.nvidia.gpu" },
+          { config: "inductor_timm", shard: 1, num_shards: 2, runner: "linux.g5.4xlarge.nvidia.gpu" },
           { config: "inductor_timm", shard: 2, num_shards: 2, runner: "linux.g5.4xlarge.nvidia.gpu" },
           { config: "inductor_torchbench", shard: 1, num_shards: 2, runner: "linux.g5.4xlarge.nvidia.gpu" },
+          { config: "inductor_torchbench", shard: 2, num_shards: 2, runner: "linux.g5.4xlarge.nvidia.gpu" },
           { config: "dynamic_inductor_huggingface", shard: 1, num_shards: 1, runner: "linux.g5.4xlarge.nvidia.gpu" },
+          { config: "dynamic_inductor_timm", shard: 1, num_shards: 2, runner: "linux.g5.4xlarge.nvidia.gpu" },
           { config: "dynamic_inductor_timm", shard: 2, num_shards: 2, runner: "linux.g5.4xlarge.nvidia.gpu" },
           { config: "dynamic_inductor_torchbench", shard: 1, num_shards: 2, runner: "linux.g5.4xlarge.nvidia.gpu" },
+          { config: "dynamic_inductor_torchbench", shard: 2, num_shards: 2, runner: "linux.g5.4xlarge.nvidia.gpu" },
           { config: "aot_inductor_huggingface", shard: 1, num_shards: 1, runner: "linux.g5.4xlarge.nvidia.gpu" },
           { config: "aot_inductor_timm", shard: 1, num_shards: 2, runner: "linux.g5.4xlarge.nvidia.gpu" },
           { config: "aot_inductor_timm", shard: 2, num_shards: 2, runner: "linux.g5.4xlarge.nvidia.gpu" },
@@ -192,6 +196,18 @@ jobs:
           { config: "inductor", shard: 1, num_shards: 1, runner: "linux.g5.4xlarge.nvidia.gpu" },
         ]}
 
+  linux-focal-cuda12_4-py3_10-gcc9-inductor-test-gcp:
+    name: cuda12.4-py3.10-gcc9-sm80
+    uses: ./.github/workflows/_linux-test.yml
+    needs: linux-focal-cuda12_4-py3_10-gcc9-inductor-build-gcp
+    with:
+      build-environment: linux-focal-cuda12.4-py3.10-gcc9-sm80
+      docker-image: ${{ needs.linux-focal-cuda12_4-py3_10-gcc9-inductor-build-gcp.outputs.docker-image }}
+      test-matrix: ${{ needs.linux-focal-cuda12_4-py3_10-gcc9-inductor-build-gcp.outputs.test-matrix }}
+      use-gha: anything-non-empty-to-use-gha
+    secrets:
+      HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }}
+
   linux-focal-cuda12_4-py3_12-gcc9-inductor-test:
     name: cuda12.4-py3.12-gcc9-sm86
     uses: ./.github/workflows/_linux-test.yml

benchmarks/dynamo/ci_expected_accuracy/cu124/aot_eager_huggingface_inference.csv

@@ -0,0 +1,185 @@
+name,accuracy,graph_breaks
+
+
+
+AlbertForMaskedLM,pass,0
+
+
+
+AlbertForQuestionAnswering,pass,0
+
+
+
+AllenaiLongformerBase,pass,4
+
+
+
+BartForCausalLM,pass,0
+
+
+
+BartForConditionalGeneration,pass,0
+
+
+
+BertForMaskedLM,pass,0
+
+
+
+BertForQuestionAnswering,pass,0
+
+
+
+BlenderbotForCausalLM,pass_due_to_skip,0
+
+
+
+BlenderbotSmallForCausalLM,pass,0
+
+
+
+BlenderbotSmallForConditionalGeneration,pass,0
+
+
+
+CamemBert,pass,0
+
+
+
+DebertaForMaskedLM,pass,0
+
+
+
+DebertaForQuestionAnswering,pass,0
+
+
+
+DebertaV2ForMaskedLM,pass_due_to_skip,0
+
+
+
+DebertaV2ForQuestionAnswering,pass,0
+
+
+
+DistilBertForMaskedLM,pass,0
+
+
+
+DistilBertForQuestionAnswering,pass,0
+
+
+
+DistillGPT2,pass,0
+
+
+
+ElectraForCausalLM,pass,0
+
+
+
+ElectraForQuestionAnswering,pass,0
+
+
+
+GPT2ForSequenceClassification,pass,2
+
+
+
+GoogleFnet,pass,0
+
+
+
+LayoutLMForMaskedLM,pass,0
+
+
+
+LayoutLMForSequenceClassification,pass,2
+
+
+
+M2M100ForConditionalGeneration,pass,0
+
+
+
+MBartForCausalLM,pass,0
+
+
+
+MBartForConditionalGeneration,pass,0
+
+
+
+MT5ForConditionalGeneration,pass,0
+
+
+
+MegatronBertForCausalLM,pass,0
+
+
+
+MegatronBertForQuestionAnswering,pass,0
+
+
+
+MobileBertForMaskedLM,pass,0
+
+
+
+MobileBertForQuestionAnswering,pass,0
+
+
+
+OPTForCausalLM,pass,0
+
+
+
+PLBartForCausalLM,pass,0
+
+
+
+PLBartForConditionalGeneration,pass,0
+
+
+
+PegasusForCausalLM,pass,0
+
+
+
+PegasusForConditionalGeneration,pass,0
+
+
+
+RobertaForCausalLM,pass,0
+
+
+
+RobertaForQuestionAnswering,pass,0
+
+
+
+Speech2Text2ForCausalLM,pass,0
+
+
+
+T5ForConditionalGeneration,pass,0
+
+
+
+T5Small,pass,0
+
+
+
+TrOCRForCausalLM,pass,0
+
+
+
+XGLMForCausalLM,pass,0
+
+
+
+XLNetLMHeadModel,pass,0
+
+
+
+YituTechConvBert,pass,0

benchmarks/dynamo/ci_expected_accuracy/cu124/aot_eager_huggingface_training.csv

@@ -0,0 +1,185 @@
+name,accuracy,graph_breaks
+
+
+
+AlbertForMaskedLM,pass,4
+
+
+
+AlbertForQuestionAnswering,pass,5
+
+
+
+AllenaiLongformerBase,pass,9
+
+
+
+BartForCausalLM,pass,12
+
+
+
+BartForConditionalGeneration,pass,24
+
+
+
+BertForMaskedLM,pass,5
+
+
+
+BertForQuestionAnswering,pass,5
+
+
+
+BlenderbotForCausalLM,eager_fail_to_run,0
+
+
+
+BlenderbotSmallForCausalLM,pass,12
+
+
+
+BlenderbotSmallForConditionalGeneration,pass,24
+
+
+
+CamemBert,pass,5
+
+
+
+DebertaForMaskedLM,pass,5
+
+
+
+DebertaForQuestionAnswering,pass,5
+
+
+
+DebertaV2ForMaskedLM,pass_due_to_skip,0
+
+
+
+DebertaV2ForQuestionAnswering,eager_1st_run_OOM,0
+
+
+
+DistilBertForMaskedLM,pass,5
+
+
+
+DistilBertForQuestionAnswering,pass,5
+
+
+
+DistillGPT2,pass,5
+
+
+
+ElectraForCausalLM,pass,4
+
+
+
+ElectraForQuestionAnswering,pass,5
+
+
+
+GPT2ForSequenceClassification,pass,7
+
+
+
+GoogleFnet,pass,5
+
+
+
+LayoutLMForMaskedLM,pass,5
+
+
+
+LayoutLMForSequenceClassification,pass,7
+
+
+
+M2M100ForConditionalGeneration,pass,4
+
+
+
+MBartForCausalLM,pass,12
+
+
+
+MBartForConditionalGeneration,pass,24
+
+
+
+MT5ForConditionalGeneration,pass,5
+
+
+
+MegatronBertForCausalLM,pass,5
+
+
+
+MegatronBertForQuestionAnswering,pass,5
+
+
+
+MobileBertForMaskedLM,pass,3
+
+
+
+MobileBertForQuestionAnswering,pass,3
+
+
+
+OPTForCausalLM,pass,12
+
+
+
+PLBartForCausalLM,pass,12
+
+
+
+PLBartForConditionalGeneration,pass,29
+
+
+
+PegasusForCausalLM,pass,12
+
+
+
+PegasusForConditionalGeneration,pass,23
+
+
+
+RobertaForCausalLM,pass,5
+
+
+
+RobertaForQuestionAnswering,pass,5
+
+
+
+Speech2Text2ForCausalLM,pass,12
+
+
+
+T5ForConditionalGeneration,pass,5
+
+
+
+T5Small,pass,5
+
+
+
+TrOCRForCausalLM,pass,12
+
+
+
+XGLMForCausalLM,pass,12
+
+
+
+XLNetLMHeadModel,pass,5
+
+
+
+YituTechConvBert,pass,5

benchmarks/dynamo/ci_expected_accuracy/cu124/aot_eager_timm_inference.csv

@@ -0,0 +1,245 @@
+name,accuracy,graph_breaks
+
+
+
+adv_inception_v3,pass,0
+
+
+
+beit_base_patch16_224,pass,0
+
+
+
+botnet26t_256,pass,0
+
+
+
+cait_m36_384,pass,0
+
+
+
+coat_lite_mini,pass,0
+
+
+
+convit_base,pass,0
+
+
+
+convmixer_768_32,pass,0
+
+
+
+convnext_base,pass,0
+
+
+
+crossvit_9_240,pass,0
+
+
+
+cspdarknet53,pass,0
+
+
+
+deit_base_distilled_patch16_224,pass,0
+
+
+
+dla102,pass,0
+
+
+
+dm_nfnet_f0,pass,0
+
+
+
+dpn107,pass,0
+
+
+
+eca_botnext26ts_256,pass,0
+
+
+
+eca_halonext26ts,pass,0
+
+
+
+ese_vovnet19b_dw,pass,0
+
+
+
+fbnetc_100,pass,0
+
+
+
+fbnetv3_b,pass,0
+
+
+
+gernet_l,pass,0
+
+
+
+ghostnet_100,pass,0
+
+
+
+gluon_inception_v3,pass,0
+
+
+
+gmixer_24_224,pass,0
+
+
+
+gmlp_s16_224,pass,0
+
+
+
+hrnet_w18,pass,0
+
+
+
+inception_v3,pass,0
+
+
+
+jx_nest_base,pass,0
+
+
+
+lcnet_050,pass,0
+
+
+
+levit_128,pass,0
+
+
+
+mixer_b16_224,pass,0
+
+
+
+mixnet_l,pass,0
+
+
+
+mnasnet_100,pass,0
+
+
+
+mobilenetv2_100,pass,0
+
+
+
+mobilenetv3_large_100,pass,0
+
+
+
+mobilevit_s,pass,0
+
+
+
+nfnet_l0,pass,0
+
+
+
+pit_b_224,pass,0
+
+
+
+pnasnet5large,pass,0
+
+
+
+poolformer_m36,pass,0
+
+
+
+regnety_002,pass,0
+
+
+
+repvgg_a2,pass,0
+
+
+
+res2net101_26w_4s,pass,0
+
+
+
+res2net50_14w_8s,pass,0
+
+
+
+res2next50,pass,0
+
+
+
+resmlp_12_224,pass,0
+
+
+
+resnest101e,pass,0
+
+
+
+rexnet_100,pass,0
+
+
+
+sebotnet33ts_256,pass,0
+
+
+
+selecsls42b,pass,0
+
+
+
+spnasnet_100,pass,0
+
+
+
+swin_base_patch4_window7_224,pass,0
+
+
+
+swsl_resnext101_32x16d,pass,0
+
+
+
+tf_efficientnet_b0,pass,0
+
+
+
+tf_mixnet_l,pass,0
+
+
+
+tinynet_a,pass,0
+
+
+
+tnt_s_patch16_224,pass,0
+
+
+
+twins_pcpvt_base,pass,0
+
+
+
+visformer_small,pass,0
+
+
+
+vit_base_patch16_224,pass,0
+
+
+
+volo_d1_224,pass,0
+
+
+
+xcit_large_24_p8_224,pass,0

benchmarks/dynamo/ci_expected_accuracy/cu124/aot_eager_timm_training.csv

@@ -0,0 +1,245 @@
+name,accuracy,graph_breaks
+
+
+
+adv_inception_v3,pass,6
+
+
+
+beit_base_patch16_224,pass,7
+
+
+
+botnet26t_256,pass,6
+
+
+
+cait_m36_384,eager_fail_to_run,0
+
+
+
+coat_lite_mini,pass,6
+
+
+
+convit_base,pass,7
+
+
+
+convmixer_768_32,pass,5
+
+
+
+convnext_base,pass,7
+
+
+
+crossvit_9_240,pass,7
+
+
+
+cspdarknet53,pass,7
+
+
+
+deit_base_distilled_patch16_224,pass,7
+
+
+
+dla102,pass,7
+
+
+
+dm_nfnet_f0,pass,6
+
+
+
+dpn107,pass,6
+
+
+
+eca_botnext26ts_256,pass,7
+
+
+
+eca_halonext26ts,pass,7
+
+
+
+ese_vovnet19b_dw,pass,7
+
+
+
+fbnetc_100,pass,7
+
+
+
+fbnetv3_b,pass,6
+
+
+
+gernet_l,pass,6
+
+
+
+ghostnet_100,pass,6
+
+
+
+gluon_inception_v3,pass,7
+
+
+
+gmixer_24_224,pass,6
+
+
+
+gmlp_s16_224,pass,7
+
+
+
+hrnet_w18,pass,5
+
+
+
+inception_v3,pass,6
+
+
+
+jx_nest_base,pass,7
+
+
+
+lcnet_050,fail_accuracy,6
+
+
+
+levit_128,pass,7
+
+
+
+mixer_b16_224,pass,7
+
+
+
+mixnet_l,pass,6
+
+
+
+mnasnet_100,pass,7
+
+
+
+mobilenetv2_100,pass,7
+
+
+
+mobilenetv3_large_100,pass,7
+
+
+
+mobilevit_s,pass,6
+
+
+
+nfnet_l0,pass,7
+
+
+
+pit_b_224,pass,6
+
+
+
+pnasnet5large,pass,5
+
+
+
+poolformer_m36,pass,6
+
+
+
+regnety_002,pass,6
+
+
+
+repvgg_a2,pass,7
+
+
+
+res2net101_26w_4s,pass,6
+
+
+
+res2net50_14w_8s,pass,6
+
+
+
+res2next50,pass,6
+
+
+
+resmlp_12_224,pass,6
+
+
+
+resnest101e,pass,6
+
+
+
+rexnet_100,pass,7
+
+
+
+sebotnet33ts_256,pass,6
+
+
+
+selecsls42b,pass,6
+
+
+
+spnasnet_100,pass,7
+
+
+
+swin_base_patch4_window7_224,pass,7
+
+
+
+swsl_resnext101_32x16d,pass,6
+
+
+
+tf_efficientnet_b0,pass,6
+
+
+
+tf_mixnet_l,pass,6
+
+
+
+tinynet_a,pass,6
+
+
+
+tnt_s_patch16_224,pass,7
+
+
+
+twins_pcpvt_base,pass,7
+
+
+
+visformer_small,pass,7
+
+
+
+vit_base_patch16_224,pass,7
+
+
+
+volo_d1_224,pass,7
+
+
+
+xcit_large_24_p8_224,pass_due_to_skip,7

benchmarks/dynamo/ci_expected_accuracy/cu124/aot_eager_torchbench_inference.csv

@@ -0,0 +1,381 @@
+name,accuracy,graph_breaks
+
+
+
+torchrec_dlrm,eager_fail_to_run,0
+
+
+
+BERT_pytorch,pass,0
+
+
+
+Background_Matting,pass_due_to_skip,0
+
+
+
+DALLE2_pytorch,pass,12
+
+
+
+LearningToPaint,pass,0
+
+
+
+Super_SloMo,pass,0
+
+
+
+alexnet,pass,0
+
+
+
+basic_gnn_edgecnn,pass,0
+
+
+
+basic_gnn_gcn,pass,6
+
+
+
+basic_gnn_gin,pass,0
+
+
+
+basic_gnn_sage,pass,0
+
+
+
+cm3leon_generate,pass,4
+
+
+
+dcgan,pass,0
+
+
+
+demucs,pass,3
+
+
+
+densenet121,pass,0
+
+
+
+detectron2_fasterrcnn_r_101_c4,eager_fail_to_run,0
+
+
+
+detectron2_fasterrcnn_r_101_dc5,eager_fail_to_run,0
+
+
+
+detectron2_fasterrcnn_r_101_fpn,eager_fail_to_run,0
+
+
+
+detectron2_fasterrcnn_r_50_c4,eager_fail_to_run,0
+
+
+
+detectron2_fasterrcnn_r_50_dc5,eager_fail_to_run,0
+
+
+
+detectron2_fasterrcnn_r_50_fpn,eager_fail_to_run,0
+
+
+
+detectron2_fcos_r_50_fpn,pass,21
+
+
+
+detectron2_maskrcnn_r_101_c4,eager_fail_to_run,0
+
+
+
+detectron2_maskrcnn_r_101_fpn,eager_fail_to_run,0
+
+
+
+detectron2_maskrcnn_r_50_c4,eager_fail_to_run,0
+
+
+
+detectron2_maskrcnn_r_50_fpn,eager_fail_to_run,0
+
+
+
+dlrm,pass,0
+
+
+
+doctr_det_predictor,pass,5
+
+
+
+doctr_reco_predictor,pass,4
+
+
+
+drq,pass,0
+
+
+
+fastNLP_Bert,pass,4
+
+
+
+functorch_dp_cifar10,pass,0
+
+
+
+functorch_maml_omniglot,pass,0
+
+
+
+hf_Albert,pass,0
+
+
+
+hf_Bart,pass,0
+
+
+
+hf_Bert,pass,0
+
+
+
+hf_Bert_large,pass,0
+
+
+
+hf_BigBird,pass,46
+
+
+
+hf_DistilBert,pass,0
+
+
+
+hf_GPT2,pass,0
+
+
+
+hf_GPT2_large,pass_due_to_skip,0
+
+
+
+hf_Reformer,pass,5
+
+
+
+hf_T5,pass,0
+
+
+
+hf_T5_base,eager_fail_to_run,0
+
+
+
+hf_T5_generate,pass,5
+
+
+
+hf_T5_large,pass_due_to_skip,0
+
+
+
+hf_Whisper,pass,0
+
+
+
+hf_distil_whisper,pass,0
+
+
+
+lennard_jones,pass,0
+
+
+
+llama,pass,0
+
+
+
+llama_v2_7b_16h,model_fail_to_load,0
+
+
+
+llava,model_fail_to_load,0
+
+
+
+maml,pass_due_to_skip,0
+
+
+
+maml_omniglot,pass,0
+
+
+
+mnasnet1_0,pass,0
+
+
+
+mobilenet_v2,pass,0
+
+
+
+mobilenet_v2_quantized_qat,model_fail_to_load,0
+
+
+
+mobilenet_v3_large,pass,0
+
+
+
+moco,pass,5
+
+
+
+moondream,model_fail_to_load,0
+
+
+
+nanogpt,pass,0
+
+
+
+nvidia_deeprecommender,pass,0
+
+
+
+opacus_cifar10,pass,0
+
+
+
+phlippe_densenet,pass,0
+
+
+
+phlippe_resnet,pass,0
+
+
+
+pyhpc_equation_of_state,pass,0
+
+
+
+pyhpc_isoneutral_mixing,pass,0
+
+
+
+pyhpc_turbulent_kinetic_energy,pass,0
+
+
+
+pytorch_CycleGAN_and_pix2pix,pass,0
+
+
+
+pytorch_stargan,pass,0
+
+
+
+pytorch_unet,pass,0
+
+
+
+resnet152,pass,0
+
+
+
+resnet18,pass,0
+
+
+
+resnet50,pass,0
+
+
+
+resnet50_quantized_qat,model_fail_to_load,0
+
+
+
+resnext50_32x4d,pass,0
+
+
+
+sam,pass,0
+
+
+
+sam_fast,pass,0
+
+
+
+shufflenet_v2_x1_0,pass,0
+
+
+
+soft_actor_critic,pass,0
+
+
+
+speech_transformer,pass,10
+
+
+
+squeezenet1_1,pass,0
+
+
+
+stable_diffusion_text_encoder,pass,0
+
+
+
+stable_diffusion_unet,pass_due_to_skip,0
+
+
+
+timm_efficientnet,pass,0
+
+
+
+timm_regnet,pass,0
+
+
+
+timm_resnest,pass,0
+
+
+
+timm_vision_transformer,pass,0
+
+
+
+timm_vision_transformer_large,pass_due_to_skip,0
+
+
+
+timm_vovnet,pass,0
+
+
+
+torch_multimodal_clip,pass,0
+
+
+
+tts_angular,pass,2
+
+
+
+vgg16,pass,0
+
+
+
+vision_maskrcnn,pass,17
+
+
+
+yolov3,pass,2

benchmarks/dynamo/ci_expected_accuracy/cu124/aot_eager_torchbench_training.csv

@@ -0,0 +1,289 @@
+name,accuracy,graph_breaks
+
+
+
+torchrec_dlrm,pass,6
+
+
+
+BERT_pytorch,pass,6
+
+
+
+Background_Matting,pass_due_to_skip,0
+
+
+
+DALLE2_pytorch,eager_fail_to_run,0
+
+
+
+LearningToPaint,pass,6
+
+
+
+Super_SloMo,pass,7
+
+
+
+alexnet,pass,6
+
+
+
+basic_gnn_edgecnn,pass,22
+
+
+
+basic_gnn_gcn,pass,13
+
+
+
+basic_gnn_gin,pass,7
+
+
+
+basic_gnn_sage,pass,7
+
+
+
+dcgan,pass,6
+
+
+
+demucs,pass,9
+
+
+
+densenet121,pass,6
+
+
+
+detectron2_maskrcnn_r_50_c4,eager_fail_to_run,0
+
+
+
+dlrm,pass,6
+
+
+
+drq,pass,6
+
+
+
+fastNLP_Bert,pass,10
+
+
+
+functorch_dp_cifar10,pass,7
+
+
+
+functorch_maml_omniglot,pass,7
+
+
+
+hf_Albert,pass,6
+
+
+
+hf_Bart,pass,6
+
+
+
+hf_Bert,pass,6
+
+
+
+hf_Bert_large,pass,6
+
+
+
+hf_BigBird,pass, 52
+
+
+
+hf_DistilBert,pass,6
+
+
+
+hf_GPT2,pass,6
+
+
+
+hf_GPT2_large,pass_due_to_skip,0
+
+
+
+hf_Reformer,pass,26
+
+
+
+hf_T5_base,eager_2nd_run_OOM,0
+
+
+
+hf_T5_large,pass_due_to_skip,0
+
+
+
+hf_Whisper,pass,6
+
+
+
+hf_distil_whisper,model_fail_to_load,0
+
+
+
+lennard_jones,pass,7
+
+
+
+llava,model_fail_to_load,0
+
+
+
+maml_omniglot,pass,7
+
+
+
+mnasnet1_0,pass,7
+
+
+
+mobilenet_v2,pass,6
+
+
+
+mobilenet_v2_quantized_qat,eager_fail_to_run,0
+
+
+
+mobilenet_v3_large,pass,7
+
+
+
+moco,pass,11
+
+
+
+nanogpt,pass,7
+
+
+
+nvidia_deeprecommender,pass,7
+
+
+
+opacus_cifar10,eager_fail_to_run,0
+
+
+
+phlippe_densenet,pass,6
+
+
+
+phlippe_resnet,pass,6
+
+
+
+pytorch_CycleGAN_and_pix2pix,pass,6
+
+
+
+pytorch_stargan,pass,6
+
+
+
+pytorch_unet,pass_due_to_skip,7
+
+
+
+resnet152,pass,7
+
+
+
+resnet18,pass,6
+
+
+
+resnet50,pass,6
+
+
+
+resnet50_quantized_qat,eager_fail_to_run,0
+
+
+
+resnext50_32x4d,pass,7
+
+
+
+sam,eager_fail_to_run,0
+
+
+
+shufflenet_v2_x1_0,pass,6
+
+
+
+soft_actor_critic,pass,6
+
+
+
+speech_transformer,pass,16
+
+
+
+squeezenet1_1,pass,6
+
+
+
+stable_diffusion_text_encoder,pass,5
+
+
+
+stable_diffusion_unet,pass_due_to_skip,0
+
+
+
+timm_efficientnet,pass,7
+
+
+
+timm_regnet,pass,6
+
+
+
+timm_resnest,pass,7
+
+
+
+timm_vision_transformer,pass,6
+
+
+
+timm_vision_transformer_large,pass_due_to_skip,0
+
+
+
+timm_vovnet,pass,6
+
+
+
+torch_multimodal_clip,pass,7
+
+
+
+tts_angular,pass,9
+
+
+
+vgg16,pass,6
+
+
+
+vision_maskrcnn,pass,34
+
+
+
+yolov3,pass,9

benchmarks/dynamo/ci_expected_accuracy/cu124/aot_inductor_huggingface_inference.csv

@@ -0,0 +1,185 @@
+name,accuracy,graph_breaks
+
+
+
+AlbertForMaskedLM,pass,0
+
+
+
+AlbertForQuestionAnswering,pass,0
+
+
+
+AllenaiLongformerBase,fail_to_run,0
+
+
+
+BartForCausalLM,pass,0
+
+
+
+BartForConditionalGeneration,pass,0
+
+
+
+BertForMaskedLM,pass,0
+
+
+
+BertForQuestionAnswering,pass,0
+
+
+
+BlenderbotForCausalLM,pass_due_to_skip,0
+
+
+
+BlenderbotSmallForCausalLM,pass,0
+
+
+
+BlenderbotSmallForConditionalGeneration,pass,0
+
+
+
+CamemBert,pass,0
+
+
+
+DebertaForMaskedLM,pass,0
+
+
+
+DebertaForQuestionAnswering,pass,0
+
+
+
+DebertaV2ForMaskedLM,pass_due_to_skip,0
+
+
+
+DebertaV2ForQuestionAnswering,pass,0
+
+
+
+DistilBertForMaskedLM,pass,0
+
+
+
+DistilBertForQuestionAnswering,pass,0
+
+
+
+DistillGPT2,pass,0
+
+
+
+ElectraForCausalLM,pass,0
+
+
+
+ElectraForQuestionAnswering,pass,0
+
+
+
+GPT2ForSequenceClassification,pass,0
+
+
+
+GoogleFnet,pass,0
+
+
+
+LayoutLMForMaskedLM,pass,0
+
+
+
+LayoutLMForSequenceClassification,pass,0
+
+
+
+M2M100ForConditionalGeneration,pass,0
+
+
+
+MBartForCausalLM,pass,0
+
+
+
+MBartForConditionalGeneration,pass,0
+
+
+
+MT5ForConditionalGeneration,pass,0
+
+
+
+MegatronBertForCausalLM,pass,0
+
+
+
+MegatronBertForQuestionAnswering,pass,0
+
+
+
+MobileBertForMaskedLM,pass,0
+
+
+
+MobileBertForQuestionAnswering,pass,0
+
+
+
+OPTForCausalLM,pass,0
+
+
+
+PLBartForCausalLM,pass,0
+
+
+
+PLBartForConditionalGeneration,pass,0
+
+
+
+PegasusForCausalLM,pass,0
+
+
+
+PegasusForConditionalGeneration,pass,0
+
+
+
+RobertaForCausalLM,pass,0
+
+
+
+RobertaForQuestionAnswering,pass,0
+
+
+
+Speech2Text2ForCausalLM,pass,0
+
+
+
+T5ForConditionalGeneration,pass,0
+
+
+
+T5Small,pass,0
+
+
+
+TrOCRForCausalLM,pass,0
+
+
+
+XGLMForCausalLM,pass,0
+
+
+
+XLNetLMHeadModel,pass,0
+
+
+
+YituTechConvBert,pass,0

benchmarks/dynamo/ci_expected_accuracy/cu124/aot_inductor_timm_inference.csv

@@ -0,0 +1,245 @@
+name,accuracy,graph_breaks
+
+
+
+adv_inception_v3,pass,0
+
+
+
+beit_base_patch16_224,pass,0
+
+
+
+botnet26t_256,pass,0
+
+
+
+cait_m36_384,pass,0
+
+
+
+coat_lite_mini,pass,0
+
+
+
+convit_base,fail_to_run,0
+
+
+
+convmixer_768_32,pass,0
+
+
+
+convnext_base,pass,0
+
+
+
+crossvit_9_240,pass,0
+
+
+
+cspdarknet53,pass,0
+
+
+
+deit_base_distilled_patch16_224,pass,0
+
+
+
+dla102,pass,0
+
+
+
+dm_nfnet_f0,pass,0
+
+
+
+dpn107,pass,0
+
+
+
+eca_botnext26ts_256,pass,0
+
+
+
+eca_halonext26ts,pass,0
+
+
+
+ese_vovnet19b_dw,pass,0
+
+
+
+fbnetc_100,pass,0
+
+
+
+fbnetv3_b,pass,0
+
+
+
+gernet_l,pass,0
+
+
+
+ghostnet_100,pass,0
+
+
+
+gluon_inception_v3,pass,0
+
+
+
+gmixer_24_224,pass,0
+
+
+
+gmlp_s16_224,pass,0
+
+
+
+hrnet_w18,pass,0
+
+
+
+inception_v3,pass,0
+
+
+
+jx_nest_base,pass,0
+
+
+
+lcnet_050,pass,0
+
+
+
+levit_128,pass,0
+
+
+
+mixer_b16_224,pass,0
+
+
+
+mixnet_l,pass,0
+
+
+
+mnasnet_100,pass,0
+
+
+
+mobilenetv2_100,pass,0
+
+
+
+mobilenetv3_large_100,pass,0
+
+
+
+mobilevit_s,pass,0
+
+
+
+nfnet_l0,pass,0
+
+
+
+pit_b_224,pass,0
+
+
+
+pnasnet5large,pass,0
+
+
+
+poolformer_m36,pass,0
+
+
+
+regnety_002,pass,0
+
+
+
+repvgg_a2,pass,0
+
+
+
+res2net101_26w_4s,pass,0
+
+
+
+res2net50_14w_8s,pass,0
+
+
+
+res2next50,pass,0
+
+
+
+resmlp_12_224,pass,0
+
+
+
+resnest101e,pass,0
+
+
+
+rexnet_100,pass,0
+
+
+
+sebotnet33ts_256,pass,0
+
+
+
+selecsls42b,pass,0
+
+
+
+spnasnet_100,pass,0
+
+
+
+swin_base_patch4_window7_224,pass,0
+
+
+
+swsl_resnext101_32x16d,pass,0
+
+
+
+tf_efficientnet_b0,pass,0
+
+
+
+tf_mixnet_l,pass,0
+
+
+
+tinynet_a,pass,0
+
+
+
+tnt_s_patch16_224,pass,0
+
+
+
+twins_pcpvt_base,pass,0
+
+
+
+visformer_small,pass,0
+
+
+
+vit_base_patch16_224,pass,0
+
+
+
+volo_d1_224,pass,0
+
+
+
+xcit_large_24_p8_224,pass,0

benchmarks/dynamo/ci_expected_accuracy/cu124/aot_inductor_torchbench_inference.csv

@@ -0,0 +1,353 @@
+name,accuracy,graph_breaks
+
+
+
+torchrec_dlrm,eager_fail_to_run,0
+
+
+
+BERT_pytorch,fail_to_run,0
+
+
+
+Background_Matting,pass_due_to_skip,0
+
+
+
+DALLE2_pytorch,fail_to_run,0
+
+
+
+LearningToPaint,pass,0
+
+
+
+Super_SloMo,pass,0
+
+
+
+alexnet,pass,0
+
+
+
+basic_gnn_edgecnn,pass,0
+
+
+
+basic_gnn_gcn,fail_to_run,0
+
+
+
+basic_gnn_gin,pass,0
+
+
+
+basic_gnn_sage,pass,0
+
+
+
+dcgan,pass,0
+
+
+
+demucs,pass,0
+
+
+
+densenet121,pass,0
+
+
+
+detectron2_fasterrcnn_r_101_c4,eager_fail_to_run,0
+
+
+
+detectron2_fasterrcnn_r_101_dc5,eager_fail_to_run,0
+
+
+
+detectron2_fasterrcnn_r_101_fpn,eager_fail_to_run,0
+
+
+
+detectron2_fasterrcnn_r_50_c4,eager_fail_to_run,0
+
+
+
+detectron2_fasterrcnn_r_50_dc5,eager_fail_to_run,0
+
+
+
+detectron2_fasterrcnn_r_50_fpn,eager_fail_to_run,0
+
+
+
+detectron2_maskrcnn_r_101_c4,eager_fail_to_run,0
+
+
+
+detectron2_maskrcnn_r_101_fpn,eager_fail_to_run,0
+
+
+
+detectron2_maskrcnn_r_50_c4,eager_fail_to_run,0
+
+
+
+detectron2_maskrcnn_r_50_fpn,eager_fail_to_run,0
+
+
+
+dlrm,pass,0
+
+
+
+doctr_det_predictor,fail_to_run,0
+
+
+
+doctr_reco_predictor,fail_to_run,0
+
+
+
+drq,fail_to_run,0
+
+
+
+functorch_dp_cifar10,pass,0
+
+
+
+functorch_maml_omniglot,pass,0
+
+
+
+hf_Albert,pass,0
+
+
+
+hf_Bart,pass,0
+
+
+
+hf_Bert,pass,0
+
+
+
+hf_Bert_large,pass,0
+
+
+
+hf_BigBird,fail_to_run,0
+
+
+
+hf_DistilBert,pass,0
+
+
+
+hf_GPT2,pass,0
+
+
+
+hf_GPT2_large,pass_due_to_skip,0
+
+
+
+hf_T5,pass,0
+
+
+
+hf_T5_base,eager_fail_to_run,0
+
+
+
+hf_T5_large,pass_due_to_skip,0
+
+
+
+hf_Whisper,pass,0
+
+
+
+hf_distil_whisper,pass,0
+
+
+
+lennard_jones,pass,0
+
+
+
+llama,fail_to_run,0
+
+
+
+llama_v2_7b_16h,model_fail_to_load,0
+
+
+
+llava,model_fail_to_load,0
+
+
+
+maml,pass_due_to_skip,0
+
+
+
+maml_omniglot,pass,0
+
+
+
+mnasnet1_0,pass,0
+
+
+
+mobilenet_v2,pass,0
+
+
+
+mobilenet_v2_quantized_qat,model_fail_to_load,0
+
+
+
+mobilenet_v3_large,pass,0
+
+
+
+moco,fail_to_run,0
+
+
+
+moondream,model_fail_to_load,0
+
+
+
+nanogpt,pass,0
+
+
+
+nvidia_deeprecommender,pass,0
+
+
+
+phlippe_densenet,pass,0
+
+
+
+phlippe_resnet,pass,0
+
+
+
+pyhpc_equation_of_state,pass,0
+
+
+
+pyhpc_isoneutral_mixing,fail_to_run,0
+
+
+
+pyhpc_turbulent_kinetic_energy,pass,0
+
+
+
+pytorch_CycleGAN_and_pix2pix,pass,0
+
+
+
+pytorch_stargan,pass,0
+
+
+
+pytorch_unet,pass,0
+
+
+
+resnet152,pass,0
+
+
+
+resnet18,pass,0
+
+
+
+resnet50,pass,0
+
+
+
+resnet50_quantized_qat,model_fail_to_load,0
+
+
+
+resnext50_32x4d,pass,0
+
+
+
+sam,fail_to_run,0
+
+
+
+sam_fast,fail_to_run,0
+
+
+
+shufflenet_v2_x1_0,pass,0
+
+
+
+soft_actor_critic,fail_to_run,0
+
+
+
+squeezenet1_1,pass,0
+
+
+
+stable_diffusion_text_encoder,pass,0
+
+
+
+stable_diffusion_unet,pass_due_to_skip,0
+
+
+
+timm_efficientnet,pass,0
+
+
+
+timm_regnet,pass,0
+
+
+
+timm_resnest,pass,0
+
+
+
+timm_vision_transformer,pass,0
+
+
+
+timm_vision_transformer_large,pass_due_to_skip,0
+
+
+
+timm_vovnet,pass,0
+
+
+
+torch_multimodal_clip,pass,0
+
+
+
+tts_angular,fail_to_run,0
+
+
+
+vgg16,pass,0
+
+
+
+vision_maskrcnn,fail_to_run,0
+
+
+
+yolov3,fail_to_run,0

benchmarks/dynamo/ci_expected_accuracy/cu124/cpu_inductor_huggingface_inference.csv

@@ -0,0 +1,185 @@
+name,accuracy,graph_breaks
+
+
+
+AlbertForMaskedLM,pass,0
+
+
+
+AlbertForQuestionAnswering,pass,0
+
+
+
+AllenaiLongformerBase,pass,4
+
+
+
+BartForCausalLM,pass,0
+
+
+
+BartForConditionalGeneration,pass,0
+
+
+
+BertForMaskedLM,pass,0
+
+
+
+BertForQuestionAnswering,pass,0
+
+
+
+BlenderbotForCausalLM,pass_due_to_skip,0
+
+
+
+BlenderbotSmallForCausalLM,pass,0
+
+
+
+BlenderbotSmallForConditionalGeneration,pass,0
+
+
+
+CamemBert,pass,0
+
+
+
+DebertaForMaskedLM,pass,0
+
+
+
+DebertaForQuestionAnswering,pass,0
+
+
+
+DebertaV2ForMaskedLM,pass_due_to_skip,0
+
+
+
+DebertaV2ForQuestionAnswering,pass,0
+
+
+
+DistilBertForMaskedLM,pass,0
+
+
+
+DistilBertForQuestionAnswering,pass,0
+
+
+
+DistillGPT2,pass,0
+
+
+
+ElectraForCausalLM,pass,0
+
+
+
+ElectraForQuestionAnswering,pass,0
+
+
+
+GPT2ForSequenceClassification,pass,2
+
+
+
+GoogleFnet,pass,0
+
+
+
+LayoutLMForMaskedLM,pass,0
+
+
+
+LayoutLMForSequenceClassification,pass,2
+
+
+
+M2M100ForConditionalGeneration,pass,0
+
+
+
+MBartForCausalLM,pass,0
+
+
+
+MBartForConditionalGeneration,pass,0
+
+
+
+MT5ForConditionalGeneration,pass,0
+
+
+
+MegatronBertForCausalLM,pass,0
+
+
+
+MegatronBertForQuestionAnswering,pass,0
+
+
+
+MobileBertForMaskedLM,pass,0
+
+
+
+MobileBertForQuestionAnswering,pass,0
+
+
+
+OPTForCausalLM,pass,0
+
+
+
+PLBartForCausalLM,pass,0
+
+
+
+PLBartForConditionalGeneration,pass,0
+
+
+
+PegasusForCausalLM,pass,0
+
+
+
+PegasusForConditionalGeneration,pass,0
+
+
+
+RobertaForCausalLM,pass,0
+
+
+
+RobertaForQuestionAnswering,pass,0
+
+
+
+Speech2Text2ForCausalLM,pass,0
+
+
+
+T5ForConditionalGeneration,pass,0
+
+
+
+T5Small,pass,0
+
+
+
+TrOCRForCausalLM,pass,0
+
+
+
+XGLMForCausalLM,pass,0
+
+
+
+XLNetLMHeadModel,pass,0
+
+
+
+YituTechConvBert,pass,0

benchmarks/dynamo/ci_expected_accuracy/cu124/cpu_inductor_timm_inference.csv

@@ -0,0 +1,245 @@
+name,accuracy,graph_breaks
+
+
+
+adv_inception_v3,pass,0
+
+
+
+beit_base_patch16_224,pass,0
+
+
+
+botnet26t_256,pass,0
+
+
+
+cait_m36_384,pass,0
+
+
+
+coat_lite_mini,pass,0
+
+
+
+convit_base,pass,0
+
+
+
+convmixer_768_32,pass,0
+
+
+
+convnext_base,pass,0
+
+
+
+crossvit_9_240,pass,0
+
+
+
+cspdarknet53,pass,0
+
+
+
+deit_base_distilled_patch16_224,pass,0
+
+
+
+dla102,pass,0
+
+
+
+dm_nfnet_f0,pass,0
+
+
+
+dpn107,pass,0
+
+
+
+eca_botnext26ts_256,pass,0
+
+
+
+eca_halonext26ts,pass,0
+
+
+
+ese_vovnet19b_dw,pass,0
+
+
+
+fbnetc_100,pass,0
+
+
+
+fbnetv3_b,pass,0
+
+
+
+gernet_l,pass,0
+
+
+
+ghostnet_100,pass,0
+
+
+
+gluon_inception_v3,pass,0
+
+
+
+gmixer_24_224,pass,0
+
+
+
+gmlp_s16_224,pass,0
+
+
+
+hrnet_w18,pass,0
+
+
+
+inception_v3,pass,0
+
+
+
+jx_nest_base,pass,0
+
+
+
+lcnet_050,pass,0
+
+
+
+levit_128,pass,0
+
+
+
+mixer_b16_224,pass,0
+
+
+
+mixnet_l,pass,0
+
+
+
+mnasnet_100,pass,0
+
+
+
+mobilenetv2_100,pass,0
+
+
+
+mobilenetv3_large_100,pass,0
+
+
+
+mobilevit_s,pass,0
+
+
+
+nfnet_l0,pass,0
+
+
+
+pit_b_224,pass,0
+
+
+
+pnasnet5large,pass,0
+
+
+
+poolformer_m36,pass,0
+
+
+
+regnety_002,pass,0
+
+
+
+repvgg_a2,pass,0
+
+
+
+res2net101_26w_4s,pass,0
+
+
+
+res2net50_14w_8s,pass,0
+
+
+
+res2next50,pass,0
+
+
+
+resmlp_12_224,pass,0
+
+
+
+resnest101e,pass,0
+
+
+
+rexnet_100,pass,0
+
+
+
+sebotnet33ts_256,pass,0
+
+
+
+selecsls42b,pass,0
+
+
+
+spnasnet_100,pass,0
+
+
+
+swin_base_patch4_window7_224,pass,0
+
+
+
+swsl_resnext101_32x16d,pass,0
+
+
+
+tf_efficientnet_b0,pass,0
+
+
+
+tf_mixnet_l,pass,0
+
+
+
+tinynet_a,pass,0
+
+
+
+tnt_s_patch16_224,pass,0
+
+
+
+twins_pcpvt_base,pass,0
+
+
+
+visformer_small,pass,0
+
+
+
+vit_base_patch16_224,pass,0
+
+
+
+volo_d1_224,pass,0
+
+
+
+xcit_large_24_p8_224,pass,0

benchmarks/dynamo/ci_expected_accuracy/cu124/cpu_inductor_torchbench_inference.csv

@@ -0,0 +1,341 @@
+name,accuracy,graph_breaks
+
+
+
+BERT_pytorch,pass,0
+
+
+
+Background_Matting,pass_due_to_skip,0
+
+
+
+DALLE2_pytorch,model_fail_to_load,0
+
+
+
+LearningToPaint,pass,0
+
+
+
+Super_SloMo,pass,0
+
+
+
+alexnet,pass,0
+
+
+
+basic_gnn_edgecnn,pass,0
+
+
+
+basic_gnn_gcn,pass,6
+
+
+
+basic_gnn_gin,pass,0
+
+
+
+basic_gnn_sage,pass,0
+
+
+
+dcgan,pass,0
+
+
+
+demucs,pass,3
+
+
+
+densenet121,pass,0
+
+
+
+detectron2_fasterrcnn_r_101_c4,pass,42
+
+
+
+detectron2_fasterrcnn_r_101_dc5,pass,42
+
+
+
+detectron2_fasterrcnn_r_101_fpn,pass,46
+
+
+
+detectron2_fasterrcnn_r_50_c4,pass,42
+
+
+
+detectron2_fasterrcnn_r_50_dc5,pass,42
+
+
+
+detectron2_fasterrcnn_r_50_fpn,pass,46
+
+
+
+detectron2_fcos_r_50_fpn,pass,23
+
+
+
+detectron2_maskrcnn_r_101_c4,fail_accuracy,57
+
+
+
+detectron2_maskrcnn_r_101_fpn,pass,64
+
+
+
+detectron2_maskrcnn_r_50_c4,pass,57
+
+
+
+detectron2_maskrcnn_r_50_fpn,pass,64
+
+
+
+dlrm,pass,0
+
+
+
+doctr_det_predictor,pass,5
+
+
+
+doctr_reco_predictor,pass,4
+
+
+
+drq,pass,0
+
+
+
+fastNLP_Bert,pass,4
+
+
+
+functorch_dp_cifar10,pass,0
+
+
+
+functorch_maml_omniglot,pass,0
+
+
+
+hf_Albert,pass,0
+
+
+
+hf_Bart,pass,0
+
+
+
+hf_Bert,pass,0
+
+
+
+hf_Bert_large,pass,0
+
+
+
+hf_DistilBert,pass,0
+
+
+
+hf_GPT2,pass,0
+
+
+
+hf_GPT2_large,pass_due_to_skip,0
+
+
+
+hf_Reformer,pass,5
+
+
+
+hf_T5_base,pass,0
+
+
+
+hf_T5_large,pass_due_to_skip,0
+
+
+
+hf_distil_whisper,pass,0
+
+
+
+lennard_jones,pass,0
+
+
+
+llama,pass,0
+
+
+
+maml,pass_due_to_skip,0
+
+
+
+maml_omniglot,pass,0
+
+
+
+mnasnet1_0,pass,0
+
+
+
+mobilenet_v2,pass,0
+
+
+
+mobilenet_v2_quantized_qat,pass,2
+
+
+
+mobilenet_v3_large,pass,0
+
+
+
+moco,model_fail_to_load,0
+
+
+
+moondream,pass,0
+
+
+
+nvidia_deeprecommender,pass,0
+
+
+
+opacus_cifar10,pass,0
+
+
+
+phlippe_densenet,pass,0
+
+
+
+phlippe_resnet,pass,0
+
+
+
+pyhpc_equation_of_state,pass,0
+
+
+
+pyhpc_isoneutral_mixing,pass,0
+
+
+
+pyhpc_turbulent_kinetic_energy,pass,0
+
+
+
+pytorch_CycleGAN_and_pix2pix,pass,0
+
+
+
+pytorch_stargan,pass,0
+
+
+
+pytorch_unet,pass,0
+
+
+
+resnet152,pass,0
+
+
+
+resnet18,pass,0
+
+
+
+resnet50,pass,0
+
+
+
+resnet50_quantized_qat,pass,2
+
+
+
+resnext50_32x4d,pass,0
+
+
+
+shufflenet_v2_x1_0,pass,0
+
+
+
+soft_actor_critic,pass,0
+
+
+
+speech_transformer,pass,10
+
+
+
+squeezenet1_1,pass,0
+
+
+
+stable_diffusion_unet,pass_due_to_skip,0
+
+
+
+timm_efficientdet,model_fail_to_load,0
+
+
+
+timm_efficientnet,pass,0
+
+
+
+timm_nfnet,pass,0
+
+
+
+timm_regnet,pass,0
+
+
+
+timm_resnest,pass,0
+
+
+
+timm_vision_transformer,pass,0
+
+
+
+timm_vision_transformer_large,pass_due_to_skip,0
+
+
+
+timm_vovnet,pass,0
+
+
+
+torch_multimodal_clip,pass,0
+
+
+
+tts_angular,pass,2
+
+
+
+vgg16,pass,0
+
+
+
+vision_maskrcnn,pass,28
+
+
+
+yolov3,pass,2

benchmarks/dynamo/ci_expected_accuracy/cu124/dynamic_aot_eager_huggingface_inference.csv

@@ -0,0 +1,185 @@
+name,accuracy,graph_breaks
+
+
+
+AlbertForMaskedLM,pass,0
+
+
+
+AlbertForQuestionAnswering,pass,0
+
+
+
+AllenaiLongformerBase,pass,4
+
+
+
+BartForCausalLM,pass,0
+
+
+
+BartForConditionalGeneration,pass,0
+
+
+
+BertForMaskedLM,pass,0
+
+
+
+BertForQuestionAnswering,pass,0
+
+
+
+BlenderbotForCausalLM,pass_due_to_skip,0
+
+
+
+BlenderbotSmallForCausalLM,pass,0
+
+
+
+BlenderbotSmallForConditionalGeneration,pass,0
+
+
+
+CamemBert,pass,0
+
+
+
+DebertaForMaskedLM,pass,0
+
+
+
+DebertaForQuestionAnswering,pass,0
+
+
+
+DebertaV2ForMaskedLM,pass_due_to_skip,0
+
+
+
+DebertaV2ForQuestionAnswering,pass,0
+
+
+
+DistilBertForMaskedLM,pass,0
+
+
+
+DistilBertForQuestionAnswering,pass,0
+
+
+
+DistillGPT2,pass,0
+
+
+
+ElectraForCausalLM,pass,0
+
+
+
+ElectraForQuestionAnswering,pass,0
+
+
+
+GPT2ForSequenceClassification,pass,2
+
+
+
+GoogleFnet,pass,0
+
+
+
+LayoutLMForMaskedLM,pass,0
+
+
+
+LayoutLMForSequenceClassification,pass,2
+
+
+
+M2M100ForConditionalGeneration,pass,0
+
+
+
+MBartForCausalLM,pass,0
+
+
+
+MBartForConditionalGeneration,pass,0
+
+
+
+MT5ForConditionalGeneration,pass,0
+
+
+
+MegatronBertForCausalLM,pass,0
+
+
+
+MegatronBertForQuestionAnswering,pass,0
+
+
+
+MobileBertForMaskedLM,pass,0
+
+
+
+MobileBertForQuestionAnswering,pass,0
+
+
+
+OPTForCausalLM,pass,0
+
+
+
+PLBartForCausalLM,pass,0
+
+
+
+PLBartForConditionalGeneration,pass,0
+
+
+
+PegasusForCausalLM,pass,0
+
+
+
+PegasusForConditionalGeneration,pass,0
+
+
+
+RobertaForCausalLM,pass,0
+
+
+
+RobertaForQuestionAnswering,pass,0
+
+
+
+Speech2Text2ForCausalLM,pass,0
+
+
+
+T5ForConditionalGeneration,pass,0
+
+
+
+T5Small,pass,0
+
+
+
+TrOCRForCausalLM,pass,0
+
+
+
+XGLMForCausalLM,pass,0
+
+
+
+XLNetLMHeadModel,pass,0
+
+
+
+YituTechConvBert,pass,0

benchmarks/dynamo/ci_expected_accuracy/cu124/dynamic_aot_eager_huggingface_training.csv

@@ -0,0 +1,185 @@
+name,accuracy,graph_breaks
+
+
+
+AlbertForMaskedLM,pass,4
+
+
+
+AlbertForQuestionAnswering,pass,5
+
+
+
+AllenaiLongformerBase,pass,9
+
+
+
+BartForCausalLM,pass,12
+
+
+
+BartForConditionalGeneration,pass,24
+
+
+
+BertForMaskedLM,pass,5
+
+
+
+BertForQuestionAnswering,pass,5
+
+
+
+BlenderbotForCausalLM,eager_fail_to_run,0
+
+
+
+BlenderbotSmallForCausalLM,pass,12
+
+
+
+BlenderbotSmallForConditionalGeneration,pass,24
+
+
+
+CamemBert,pass,5
+
+
+
+DebertaForMaskedLM,pass,5
+
+
+
+DebertaForQuestionAnswering,pass,5
+
+
+
+DebertaV2ForMaskedLM,pass_due_to_skip,0
+
+
+
+DebertaV2ForQuestionAnswering,eager_1st_run_OOM,0
+
+
+
+DistilBertForMaskedLM,pass,5
+
+
+
+DistilBertForQuestionAnswering,pass,5
+
+
+
+DistillGPT2,pass,5
+
+
+
+ElectraForCausalLM,pass,4
+
+
+
+ElectraForQuestionAnswering,pass,5
+
+
+
+GPT2ForSequenceClassification,pass,7
+
+
+
+GoogleFnet,pass,5
+
+
+
+LayoutLMForMaskedLM,pass,5
+
+
+
+LayoutLMForSequenceClassification,pass,7
+
+
+
+M2M100ForConditionalGeneration,pass,4
+
+
+
+MBartForCausalLM,pass,12
+
+
+
+MBartForConditionalGeneration,pass,24
+
+
+
+MT5ForConditionalGeneration,pass,5
+
+
+
+MegatronBertForCausalLM,pass,5
+
+
+
+MegatronBertForQuestionAnswering,pass,5
+
+
+
+MobileBertForMaskedLM,pass,3
+
+
+
+MobileBertForQuestionAnswering,pass,3
+
+
+
+OPTForCausalLM,pass,12
+
+
+
+PLBartForCausalLM,pass,12
+
+
+
+PLBartForConditionalGeneration,pass,29
+
+
+
+PegasusForCausalLM,pass,12
+
+
+
+PegasusForConditionalGeneration,pass,23
+
+
+
+RobertaForCausalLM,pass,5
+
+
+
+RobertaForQuestionAnswering,pass,5
+
+
+
+Speech2Text2ForCausalLM,pass,12
+
+
+
+T5ForConditionalGeneration,pass,5
+
+
+
+T5Small,pass,5
+
+
+
+TrOCRForCausalLM,pass,12
+
+
+
+XGLMForCausalLM,pass,12
+
+
+
+XLNetLMHeadModel,pass,5
+
+
+
+YituTechConvBert,pass,5

benchmarks/dynamo/ci_expected_accuracy/cu124/dynamic_aot_eager_timm_inference.csv

@@ -0,0 +1,245 @@
+name,accuracy,graph_breaks
+
+
+
+adv_inception_v3,pass,0
+
+
+
+beit_base_patch16_224,pass,0
+
+
+
+botnet26t_256,pass,0
+
+
+
+cait_m36_384,pass,0
+
+
+
+coat_lite_mini,pass,0
+
+
+
+convit_base,pass,0
+
+
+
+convmixer_768_32,pass,0
+
+
+
+convnext_base,pass,0
+
+
+
+crossvit_9_240,pass,0
+
+
+
+cspdarknet53,pass,0
+
+
+
+deit_base_distilled_patch16_224,pass,0
+
+
+
+dla102,pass,0
+
+
+
+dm_nfnet_f0,pass,0
+
+
+
+dpn107,pass,0
+
+
+
+eca_botnext26ts_256,pass,0
+
+
+
+eca_halonext26ts,pass,0
+
+
+
+ese_vovnet19b_dw,pass,0
+
+
+
+fbnetc_100,pass,0
+
+
+
+fbnetv3_b,pass,0
+
+
+
+gernet_l,pass,0
+
+
+
+ghostnet_100,pass,0
+
+
+
+gluon_inception_v3,pass,0
+
+
+
+gmixer_24_224,pass,0
+
+
+
+gmlp_s16_224,pass,0
+
+
+
+hrnet_w18,pass,0
+
+
+
+inception_v3,pass,0
+
+
+
+jx_nest_base,pass,0
+
+
+
+lcnet_050,pass,0
+
+
+
+levit_128,pass,0
+
+
+
+mixer_b16_224,pass,0
+
+
+
+mixnet_l,pass,0
+
+
+
+mnasnet_100,pass,0
+
+
+
+mobilenetv2_100,pass,0
+
+
+
+mobilenetv3_large_100,pass,0
+
+
+
+mobilevit_s,pass,0
+
+
+
+nfnet_l0,pass,0
+
+
+
+pit_b_224,pass,0
+
+
+
+pnasnet5large,pass,0
+
+
+
+poolformer_m36,pass,0
+
+
+
+regnety_002,pass,0
+
+
+
+repvgg_a2,pass,0
+
+
+
+res2net101_26w_4s,pass,0
+
+
+
+res2net50_14w_8s,pass,0
+
+
+
+res2next50,pass,0
+
+
+
+resmlp_12_224,pass,0
+
+
+
+resnest101e,pass,0
+
+
+
+rexnet_100,pass,0
+
+
+
+sebotnet33ts_256,pass,0
+
+
+
+selecsls42b,pass,0
+
+
+
+spnasnet_100,pass,0
+
+
+
+swin_base_patch4_window7_224,pass,0
+
+
+
+swsl_resnext101_32x16d,pass,0
+
+
+
+tf_efficientnet_b0,pass,0
+
+
+
+tf_mixnet_l,pass,0
+
+
+
+tinynet_a,pass,0
+
+
+
+tnt_s_patch16_224,pass,0
+
+
+
+twins_pcpvt_base,pass,0
+
+
+
+visformer_small,pass,0
+
+
+
+vit_base_patch16_224,pass,0
+
+
+
+volo_d1_224,pass,0
+
+
+
+xcit_large_24_p8_224,pass,0

benchmarks/dynamo/ci_expected_accuracy/cu124/dynamic_aot_eager_timm_training.csv

@@ -0,0 +1,245 @@
+name,accuracy,graph_breaks
+
+
+
+adv_inception_v3,pass,6
+
+
+
+beit_base_patch16_224,pass,7
+
+
+
+botnet26t_256,pass,6
+
+
+
+cait_m36_384,eager_fail_to_run,0
+
+
+
+coat_lite_mini,pass,6
+
+
+
+convit_base,pass,7
+
+
+
+convmixer_768_32,pass,5
+
+
+
+convnext_base,pass,7
+
+
+
+crossvit_9_240,pass,7
+
+
+
+cspdarknet53,pass,7
+
+
+
+deit_base_distilled_patch16_224,pass,7
+
+
+
+dla102,pass,7
+
+
+
+dm_nfnet_f0,pass,6
+
+
+
+dpn107,pass,6
+
+
+
+eca_botnext26ts_256,pass,7
+
+
+
+eca_halonext26ts,pass,7
+
+
+
+ese_vovnet19b_dw,pass,7
+
+
+
+fbnetc_100,pass,7
+
+
+
+fbnetv3_b,pass,6
+
+
+
+gernet_l,pass,6
+
+
+
+ghostnet_100,pass,6
+
+
+
+gluon_inception_v3,pass,7
+
+
+
+gmixer_24_224,pass,6
+
+
+
+gmlp_s16_224,pass,7
+
+
+
+hrnet_w18,pass,5
+
+
+
+inception_v3,pass,6
+
+
+
+jx_nest_base,pass,7
+
+
+
+lcnet_050,fail_accuracy,6
+
+
+
+levit_128,pass,7
+
+
+
+mixer_b16_224,pass,7
+
+
+
+mixnet_l,pass,6
+
+
+
+mnasnet_100,pass,7
+
+
+
+mobilenetv2_100,pass,7
+
+
+
+mobilenetv3_large_100,pass,7
+
+
+
+mobilevit_s,pass,6
+
+
+
+nfnet_l0,pass,7
+
+
+
+pit_b_224,pass,6
+
+
+
+pnasnet5large,pass,5
+
+
+
+poolformer_m36,pass,6
+
+
+
+regnety_002,pass,6
+
+
+
+repvgg_a2,pass,7
+
+
+
+res2net101_26w_4s,pass,6
+
+
+
+res2net50_14w_8s,pass,6
+
+
+
+res2next50,pass,6
+
+
+
+resmlp_12_224,pass,6
+
+
+
+resnest101e,pass,6
+
+
+
+rexnet_100,pass,7
+
+
+
+sebotnet33ts_256,pass,6
+
+
+
+selecsls42b,pass,6
+
+
+
+spnasnet_100,pass,7
+
+
+
+swin_base_patch4_window7_224,pass,7
+
+
+
+swsl_resnext101_32x16d,pass,6
+
+
+
+tf_efficientnet_b0,pass,6
+
+
+
+tf_mixnet_l,pass,6
+
+
+
+tinynet_a,pass,6
+
+
+
+tnt_s_patch16_224,pass,7
+
+
+
+twins_pcpvt_base,pass,7
+
+
+
+visformer_small,pass,7
+
+
+
+vit_base_patch16_224,pass,7
+
+
+
+volo_d1_224,pass,7
+
+
+
+xcit_large_24_p8_224,pass_due_to_skip,7

benchmarks/dynamo/ci_expected_accuracy/cu124/dynamic_aot_eager_torchbench_inference.csv

@@ -0,0 +1,377 @@
+name,accuracy,graph_breaks
+
+
+
+torchrec_dlrm,eager_fail_to_run,0
+
+
+
+BERT_pytorch,pass,0
+
+
+
+Background_Matting,pass_due_to_skip,0
+
+
+
+DALLE2_pytorch,pass,12
+
+
+
+LearningToPaint,pass,0
+
+
+
+Super_SloMo,pass,0
+
+
+
+alexnet,pass,0
+
+
+
+basic_gnn_edgecnn,pass,0
+
+
+
+basic_gnn_gcn,pass,6
+
+
+
+basic_gnn_gin,pass,0
+
+
+
+basic_gnn_sage,pass,0
+
+
+
+cm3leon_generate,pass,4
+
+
+
+dcgan,pass,0
+
+
+
+demucs,pass,3
+
+
+
+densenet121,pass,0
+
+
+
+detectron2_fasterrcnn_r_101_c4,eager_fail_to_run,0
+
+
+
+detectron2_fasterrcnn_r_101_dc5,eager_fail_to_run,0
+
+
+
+detectron2_fasterrcnn_r_101_fpn,eager_fail_to_run,0
+
+
+
+detectron2_fasterrcnn_r_50_c4,eager_fail_to_run,0
+
+
+
+detectron2_fasterrcnn_r_50_dc5,eager_fail_to_run,0
+
+
+
+detectron2_fasterrcnn_r_50_fpn,eager_fail_to_run,0
+
+
+
+detectron2_fcos_r_50_fpn,pass,21
+
+
+
+detectron2_maskrcnn_r_101_c4,eager_fail_to_run,0
+
+
+
+detectron2_maskrcnn_r_101_fpn,eager_fail_to_run,0
+
+
+
+detectron2_maskrcnn_r_50_c4,eager_fail_to_run,0
+
+
+
+detectron2_maskrcnn_r_50_fpn,eager_fail_to_run,0
+
+
+
+dlrm,pass,0
+
+
+
+doctr_det_predictor,pass,5
+
+
+
+doctr_reco_predictor,pass,4
+
+
+
+drq,pass,0
+
+
+
+fastNLP_Bert,pass,4
+
+
+
+functorch_dp_cifar10,pass,0
+
+
+
+functorch_maml_omniglot,pass,0
+
+
+
+hf_Albert,pass,0
+
+
+
+hf_Bart,pass,0
+
+
+
+hf_Bert,pass,0
+
+
+
+hf_Bert_large,pass,0
+
+
+
+hf_BigBird,pass,46
+
+
+
+hf_DistilBert,pass,0
+
+
+
+hf_GPT2,pass,0
+
+
+
+hf_GPT2_large,pass_due_to_skip,0
+
+
+
+hf_Reformer,pass,5
+
+
+
+hf_T5,pass,0
+
+
+
+hf_T5_base,eager_fail_to_run,0
+
+
+
+hf_T5_generate,fail_to_run,5
+
+
+
+hf_T5_large,pass_due_to_skip,0
+
+
+
+hf_Whisper,pass,0
+
+
+
+hf_distil_whisper,pass,0
+
+
+
+lennard_jones,pass,0
+
+
+
+llama,pass,0
+
+
+
+llama_v2_7b_16h,model_fail_to_load,0
+
+
+
+llava,model_fail_to_load,0
+
+
+
+maml,pass_due_to_skip,0
+
+
+
+maml_omniglot,pass,0
+
+
+
+mnasnet1_0,pass,0
+
+
+
+mobilenet_v2,pass,0
+
+
+
+mobilenet_v2_quantized_qat,model_fail_to_load,0
+
+
+
+mobilenet_v3_large,pass,0
+
+
+
+moco,pass,5
+
+
+
+moondream,model_fail_to_load,0
+
+
+
+nanogpt,pass,0
+
+
+
+nvidia_deeprecommender,pass,0
+
+
+
+opacus_cifar10,pass,0
+
+
+
+phlippe_densenet,pass,0
+
+
+
+phlippe_resnet,pass,0
+
+
+
+pyhpc_equation_of_state,pass,0
+
+
+
+pyhpc_isoneutral_mixing,pass,0
+
+
+
+pyhpc_turbulent_kinetic_energy,pass,0
+
+
+
+pytorch_CycleGAN_and_pix2pix,pass,0
+
+
+
+pytorch_stargan,pass,0
+
+
+
+pytorch_unet,pass,0
+
+
+
+resnet152,pass,0
+
+
+
+resnet18,pass,0
+
+
+
+resnet50,pass,0
+
+
+
+resnet50_quantized_qat,model_fail_to_load,0
+
+
+
+resnext50_32x4d,pass,0
+
+
+
+sam,pass,0
+
+
+
+shufflenet_v2_x1_0,pass,0
+
+
+
+soft_actor_critic,pass,0
+
+
+
+speech_transformer,pass,10
+
+
+
+squeezenet1_1,pass,0
+
+
+
+stable_diffusion_text_encoder,pass,0
+
+
+
+stable_diffusion_unet,pass_due_to_skip,0
+
+
+
+timm_efficientnet,pass,0
+
+
+
+timm_regnet,pass,0
+
+
+
+timm_resnest,pass,0
+
+
+
+timm_vision_transformer,pass,0
+
+
+
+timm_vision_transformer_large,pass_due_to_skip,0
+
+
+
+timm_vovnet,pass,0
+
+
+
+torch_multimodal_clip,pass,0
+
+
+
+tts_angular,pass,2
+
+
+
+vgg16,pass,0
+
+
+
+vision_maskrcnn,pass,17
+
+
+
+yolov3,pass,2

benchmarks/dynamo/ci_expected_accuracy/cu124/dynamic_aot_eager_torchbench_training.csv

@@ -0,0 +1,285 @@
+name,accuracy,graph_breaks
+
+
+
+torchrec_dlrm,fail_to_run,3
+
+
+
+BERT_pytorch,pass,6
+
+
+
+Background_Matting,pass_due_to_skip,0
+
+
+
+DALLE2_pytorch,eager_fail_to_run,0
+
+
+
+LearningToPaint,pass,6
+
+
+
+Super_SloMo,pass,7
+
+
+
+alexnet,pass,6
+
+
+
+basic_gnn_edgecnn,pass,22
+
+
+
+basic_gnn_gcn,pass,13
+
+
+
+basic_gnn_gin,pass,7
+
+
+
+basic_gnn_sage,pass,7
+
+
+
+dcgan,pass,6
+
+
+
+demucs,pass,9
+
+
+
+densenet121,pass,6
+
+
+
+detectron2_maskrcnn_r_50_c4,eager_fail_to_run,0
+
+
+
+dlrm,pass,6
+
+
+
+drq,pass,6
+
+
+
+fastNLP_Bert,pass,10
+
+
+
+functorch_dp_cifar10,pass,7
+
+
+
+functorch_maml_omniglot,pass,7
+
+
+
+hf_Albert,pass,6
+
+
+
+hf_Bart,pass,6
+
+
+
+hf_Bert,pass,6
+
+
+
+hf_Bert_large,pass,6
+
+
+
+hf_BigBird,pass,52
+
+
+
+hf_DistilBert,pass,6
+
+
+
+hf_GPT2,pass,6
+
+
+
+hf_GPT2_large,pass_due_to_skip,0
+
+
+
+hf_Reformer,pass,26
+
+
+
+hf_T5_base,eager_2nd_run_OOM,0
+
+
+
+hf_T5_large,pass_due_to_skip,0
+
+
+
+hf_Whisper,pass,6
+
+
+
+hf_distil_whisper,model_fail_to_load,0
+
+
+
+lennard_jones,pass,7
+
+
+
+llava,model_fail_to_load,0
+
+
+
+maml_omniglot,pass,7
+
+
+
+mnasnet1_0,pass,7
+
+
+
+mobilenet_v2,pass,6
+
+
+
+mobilenet_v2_quantized_qat,eager_fail_to_run,0
+
+
+
+mobilenet_v3_large,pass,7
+
+
+
+moco,pass,11
+
+
+
+nanogpt,pass,7
+
+
+
+nvidia_deeprecommender,pass,7
+
+
+
+opacus_cifar10,eager_fail_to_run,0
+
+
+
+phlippe_densenet,pass,6
+
+
+
+phlippe_resnet,pass,6
+
+
+
+pytorch_CycleGAN_and_pix2pix,pass,6
+
+
+
+pytorch_stargan,pass,6
+
+
+
+pytorch_unet,pass_due_to_skip,7
+
+
+
+resnet152,pass,7
+
+
+
+resnet18,pass,6
+
+
+
+resnet50,pass,6
+
+
+
+resnet50_quantized_qat,eager_fail_to_run,0
+
+
+
+resnext50_32x4d,pass,7
+
+
+
+sam,eager_fail_to_run,0
+
+
+
+shufflenet_v2_x1_0,pass,6
+
+
+
+soft_actor_critic,pass,6
+
+
+
+squeezenet1_1,pass,6
+
+
+
+stable_diffusion_text_encoder,pass,5
+
+
+
+stable_diffusion_unet,pass_due_to_skip,0
+
+
+
+timm_efficientnet,pass,7
+
+
+
+timm_regnet,pass,6
+
+
+
+timm_resnest,pass,7
+
+
+
+timm_vision_transformer,pass,6
+
+
+
+timm_vision_transformer_large,pass_due_to_skip,0
+
+
+
+timm_vovnet,pass,6
+
+
+
+torch_multimodal_clip,pass,7
+
+
+
+tts_angular,pass,9
+
+
+
+vgg16,pass,6
+
+
+
+vision_maskrcnn,pass,34
+
+
+
+yolov3,pass,9

benchmarks/dynamo/ci_expected_accuracy/cu124/dynamic_cpu_inductor_huggingface_inference.csv

@@ -0,0 +1,185 @@
+name,accuracy,graph_breaks
+
+
+
+AlbertForMaskedLM,pass,0
+
+
+
+AlbertForQuestionAnswering,pass,0
+
+
+
+AllenaiLongformerBase,pass,4
+
+
+
+BartForCausalLM,pass,0
+
+
+
+BartForConditionalGeneration,pass,0
+
+
+
+BertForMaskedLM,pass,0
+
+
+
+BertForQuestionAnswering,pass,0
+
+
+
+BlenderbotForCausalLM,pass_due_to_skip,0
+
+
+
+BlenderbotSmallForCausalLM,pass,0
+
+
+
+BlenderbotSmallForConditionalGeneration,pass,0
+
+
+
+CamemBert,pass,0
+
+
+
+DebertaForMaskedLM,pass,0
+
+
+
+DebertaForQuestionAnswering,pass,0
+
+
+
+DebertaV2ForMaskedLM,pass_due_to_skip,0
+
+
+
+DebertaV2ForQuestionAnswering,pass,0
+
+
+
+DistilBertForMaskedLM,pass,0
+
+
+
+DistilBertForQuestionAnswering,pass,0
+
+
+
+DistillGPT2,pass,0
+
+
+
+ElectraForCausalLM,pass,0
+
+
+
+ElectraForQuestionAnswering,pass,0
+
+
+
+GPT2ForSequenceClassification,pass,2
+
+
+
+GoogleFnet,pass,0
+
+
+
+LayoutLMForMaskedLM,pass,0
+
+
+
+LayoutLMForSequenceClassification,pass,2
+
+
+
+M2M100ForConditionalGeneration,pass,0
+
+
+
+MBartForCausalLM,pass,0
+
+
+
+MBartForConditionalGeneration,pass,0
+
+
+
+MT5ForConditionalGeneration,pass,0
+
+
+
+MegatronBertForCausalLM,pass,0
+
+
+
+MegatronBertForQuestionAnswering,pass,0
+
+
+
+MobileBertForMaskedLM,pass,0
+
+
+
+MobileBertForQuestionAnswering,pass,0
+
+
+
+OPTForCausalLM,pass,0
+
+
+
+PLBartForCausalLM,pass,0
+
+
+
+PLBartForConditionalGeneration,pass,0
+
+
+
+PegasusForCausalLM,pass,0
+
+
+
+PegasusForConditionalGeneration,pass,0
+
+
+
+RobertaForCausalLM,pass,0
+
+
+
+RobertaForQuestionAnswering,pass,0
+
+
+
+Speech2Text2ForCausalLM,pass,0
+
+
+
+T5ForConditionalGeneration,pass,0
+
+
+
+T5Small,pass,0
+
+
+
+TrOCRForCausalLM,pass,0
+
+
+
+XGLMForCausalLM,pass,0
+
+
+
+XLNetLMHeadModel,pass,0
+
+
+
+YituTechConvBert,pass,0

benchmarks/dynamo/ci_expected_accuracy/cu124/dynamic_cpu_inductor_timm_inference.csv

@@ -0,0 +1,245 @@
+name,accuracy,graph_breaks
+
+
+
+adv_inception_v3,pass,0
+
+
+
+beit_base_patch16_224,pass,0
+
+
+
+botnet26t_256,pass,0
+
+
+
+cait_m36_384,pass,0
+
+
+
+coat_lite_mini,pass,0
+
+
+
+convit_base,pass,0
+
+
+
+convmixer_768_32,pass,0
+
+
+
+convnext_base,pass,0
+
+
+
+crossvit_9_240,pass,0
+
+
+
+cspdarknet53,pass,0
+
+
+
+deit_base_distilled_patch16_224,pass,0
+
+
+
+dla102,pass,0
+
+
+
+dm_nfnet_f0,pass,0
+
+
+
+dpn107,pass,0
+
+
+
+eca_botnext26ts_256,pass,0
+
+
+
+eca_halonext26ts,pass,0
+
+
+
+ese_vovnet19b_dw,pass,0
+
+
+
+fbnetc_100,pass,0
+
+
+
+fbnetv3_b,pass,0
+
+
+
+gernet_l,pass,0
+
+
+
+ghostnet_100,pass,0
+
+
+
+gluon_inception_v3,pass,0
+
+
+
+gmixer_24_224,pass,0
+
+
+
+gmlp_s16_224,pass,0
+
+
+
+hrnet_w18,pass,0
+
+
+
+inception_v3,pass,0
+
+
+
+jx_nest_base,pass,0
+
+
+
+lcnet_050,pass,0
+
+
+
+levit_128,pass,0
+
+
+
+mixer_b16_224,pass,0
+
+
+
+mixnet_l,pass,0
+
+
+
+mnasnet_100,pass,0
+
+
+
+mobilenetv2_100,pass,0
+
+
+
+mobilenetv3_large_100,pass,0
+
+
+
+mobilevit_s,pass,0
+
+
+
+nfnet_l0,pass,0
+
+
+
+pit_b_224,pass,0
+
+
+
+pnasnet5large,pass,0
+
+
+
+poolformer_m36,pass,0
+
+
+
+regnety_002,pass,0
+
+
+
+repvgg_a2,pass,0
+
+
+
+res2net101_26w_4s,pass,0
+
+
+
+res2net50_14w_8s,pass,0
+
+
+
+res2next50,pass,0
+
+
+
+resmlp_12_224,pass,0
+
+
+
+resnest101e,pass,0
+
+
+
+rexnet_100,pass,0
+
+
+
+sebotnet33ts_256,pass,0
+
+
+
+selecsls42b,pass,0
+
+
+
+spnasnet_100,pass,0
+
+
+
+swin_base_patch4_window7_224,pass,0
+
+
+
+swsl_resnext101_32x16d,pass,0
+
+
+
+tf_efficientnet_b0,pass,0
+
+
+
+tf_mixnet_l,pass,0
+
+
+
+tinynet_a,pass,0
+
+
+
+tnt_s_patch16_224,pass,0
+
+
+
+twins_pcpvt_base,pass,0
+
+
+
+visformer_small,pass,0
+
+
+
+vit_base_patch16_224,pass,0
+
+
+
+volo_d1_224,pass,0
+
+
+
+xcit_large_24_p8_224,pass,0

benchmarks/dynamo/ci_expected_accuracy/cu124/dynamic_cpu_inductor_torchbench_inference.csv

@@ -0,0 +1,301 @@
+name,accuracy,graph_breaks
+
+
+
+BERT_pytorch,pass,0
+
+
+
+Background_Matting,pass_due_to_skip,0
+
+
+
+DALLE2_pytorch,model_fail_to_load,0
+
+
+
+LearningToPaint,pass,0
+
+
+
+Super_SloMo,pass,0
+
+
+
+alexnet,pass,0
+
+
+
+basic_gnn_edgecnn,pass,0
+
+
+
+basic_gnn_gcn,pass,6
+
+
+
+basic_gnn_gin,pass,0
+
+
+
+basic_gnn_sage,pass,0
+
+
+
+dcgan,pass,0
+
+
+
+demucs,pass,3
+
+
+
+densenet121,pass,0
+
+
+
+detectron2_fcos_r_50_fpn,pass,23
+
+
+
+dlrm,pass,0
+
+
+
+doctr_det_predictor,pass,5
+
+
+
+doctr_reco_predictor,pass,4
+
+
+
+drq,pass,0
+
+
+
+fastNLP_Bert,pass,4
+
+
+
+functorch_dp_cifar10,pass,0
+
+
+
+functorch_maml_omniglot,pass,0
+
+
+
+hf_Albert,pass,0
+
+
+
+hf_Bart,pass,0
+
+
+
+hf_Bert,pass,0
+
+
+
+hf_Bert_large,pass,0
+
+
+
+hf_DistilBert,pass,0
+
+
+
+hf_GPT2,pass,0
+
+
+
+hf_GPT2_large,pass_due_to_skip,0
+
+
+
+hf_Reformer,pass,5
+
+
+
+hf_T5_base,pass,0
+
+
+
+hf_T5_large,pass_due_to_skip,0
+
+
+
+hf_distil_whisper,pass,0
+
+
+
+lennard_jones,pass,0
+
+
+
+llama,pass,0
+
+
+
+maml,pass_due_to_skip,0
+
+
+
+maml_omniglot,pass,0
+
+
+
+mnasnet1_0,pass,0
+
+
+
+mobilenet_v2,pass,0
+
+
+
+mobilenet_v2_quantized_qat,pass,2
+
+
+
+mobilenet_v3_large,pass,0
+
+
+
+moco,model_fail_to_load,0
+
+
+
+moondream,pass,0
+
+
+
+nvidia_deeprecommender,pass,0
+
+
+
+opacus_cifar10,pass,0
+
+
+
+phlippe_densenet,pass,0
+
+
+
+phlippe_resnet,pass,0
+
+
+
+pyhpc_equation_of_state,pass,0
+
+
+
+pyhpc_isoneutral_mixing,pass,0
+
+
+
+pyhpc_turbulent_kinetic_energy,pass,0
+
+
+
+pytorch_CycleGAN_and_pix2pix,pass,0
+
+
+
+pytorch_stargan,pass,0
+
+
+
+pytorch_unet,pass,0
+
+
+
+resnet152,pass,0
+
+
+
+resnet18,pass,0
+
+
+
+resnet50,pass,0
+
+
+
+resnet50_quantized_qat,pass,2
+
+
+
+resnext50_32x4d,pass,0
+
+
+
+shufflenet_v2_x1_0,pass,0
+
+
+
+soft_actor_critic,pass,0
+
+
+
+speech_transformer,pass,10
+
+
+
+squeezenet1_1,pass,0
+
+
+
+stable_diffusion_unet,pass_due_to_skip,0
+
+
+
+timm_efficientdet,model_fail_to_load,0
+
+
+
+timm_efficientnet,pass,0
+
+
+
+timm_nfnet,pass,0
+
+
+
+timm_regnet,pass,0
+
+
+
+timm_resnest,pass,0
+
+
+
+timm_vision_transformer,pass,0
+
+
+
+timm_vision_transformer_large,pass_due_to_skip,0
+
+
+
+timm_vovnet,pass,0
+
+
+
+torch_multimodal_clip,pass,3
+
+
+
+tts_angular,pass,2
+
+
+
+vgg16,pass,0
+
+
+
+vision_maskrcnn,pass,28
+
+
+
+yolov3,pass,2

benchmarks/dynamo/ci_expected_accuracy/cu124/dynamic_inductor_huggingface_inference.csv

@@ -0,0 +1,185 @@
+name,accuracy,graph_breaks
+
+
+
+AlbertForMaskedLM,pass,0
+
+
+
+AlbertForQuestionAnswering,pass,0
+
+
+
+AllenaiLongformerBase,pass,4
+
+
+
+BartForCausalLM,pass,0
+
+
+
+BartForConditionalGeneration,pass,0
+
+
+
+BertForMaskedLM,pass,0
+
+
+
+BertForQuestionAnswering,pass,0
+
+
+
+BlenderbotForCausalLM,pass_due_to_skip,0
+
+
+
+BlenderbotSmallForCausalLM,pass,0
+
+
+
+BlenderbotSmallForConditionalGeneration,pass,0
+
+
+
+CamemBert,pass,0
+
+
+
+DebertaForMaskedLM,pass,0
+
+
+
+DebertaForQuestionAnswering,pass,0
+
+
+
+DebertaV2ForMaskedLM,pass_due_to_skip,0
+
+
+
+DebertaV2ForQuestionAnswering,pass,0
+
+
+
+DistilBertForMaskedLM,pass,0
+
+
+
+DistilBertForQuestionAnswering,pass,0
+
+
+
+DistillGPT2,pass,0
+
+
+
+ElectraForCausalLM,pass,0
+
+
+
+ElectraForQuestionAnswering,pass,0
+
+
+
+GPT2ForSequenceClassification,pass,2
+
+
+
+GoogleFnet,pass,0
+
+
+
+LayoutLMForMaskedLM,pass,0
+
+
+
+LayoutLMForSequenceClassification,pass,2
+
+
+
+M2M100ForConditionalGeneration,pass,0
+
+
+
+MBartForCausalLM,pass,0
+
+
+
+MBartForConditionalGeneration,pass,0
+
+
+
+MT5ForConditionalGeneration,pass,0
+
+
+
+MegatronBertForCausalLM,pass,0
+
+
+
+MegatronBertForQuestionAnswering,pass,0
+
+
+
+MobileBertForMaskedLM,pass,0
+
+
+
+MobileBertForQuestionAnswering,pass,0
+
+
+
+OPTForCausalLM,pass,0
+
+
+
+PLBartForCausalLM,pass,0
+
+
+
+PLBartForConditionalGeneration,pass,0
+
+
+
+PegasusForCausalLM,pass,0
+
+
+
+PegasusForConditionalGeneration,pass,0
+
+
+
+RobertaForCausalLM,pass,0
+
+
+
+RobertaForQuestionAnswering,pass,0
+
+
+
+Speech2Text2ForCausalLM,pass,0
+
+
+
+T5ForConditionalGeneration,pass,0
+
+
+
+T5Small,pass,0
+
+
+
+TrOCRForCausalLM,pass,0
+
+
+
+XGLMForCausalLM,pass,0
+
+
+
+XLNetLMHeadModel,pass,0
+
+
+
+YituTechConvBert,pass,0

benchmarks/dynamo/ci_expected_accuracy/cu124/dynamic_inductor_huggingface_training.csv

@@ -0,0 +1,185 @@
+name,accuracy,graph_breaks
+
+
+
+AlbertForMaskedLM,pass,4
+
+
+
+AlbertForQuestionAnswering,pass,5
+
+
+
+AllenaiLongformerBase,pass,9
+
+
+
+BartForCausalLM,pass,12
+
+
+
+BartForConditionalGeneration,pass,24
+
+
+
+BertForMaskedLM,pass,5
+
+
+
+BertForQuestionAnswering,pass,5
+
+
+
+BlenderbotForCausalLM,eager_fail_to_run,0
+
+
+
+BlenderbotSmallForCausalLM,pass,12
+
+
+
+BlenderbotSmallForConditionalGeneration,pass,24
+
+
+
+CamemBert,pass,5
+
+
+
+DebertaForMaskedLM,pass,5
+
+
+
+DebertaForQuestionAnswering,pass,5
+
+
+
+DebertaV2ForMaskedLM,pass_due_to_skip,0
+
+
+
+DebertaV2ForQuestionAnswering,eager_1st_run_OOM,0
+
+
+
+DistilBertForMaskedLM,pass,5
+
+
+
+DistilBertForQuestionAnswering,pass,5
+
+
+
+DistillGPT2,pass,5
+
+
+
+ElectraForCausalLM,pass,4
+
+
+
+ElectraForQuestionAnswering,pass,5
+
+
+
+GPT2ForSequenceClassification,pass,7
+
+
+
+GoogleFnet,pass,5
+
+
+
+LayoutLMForMaskedLM,pass,5
+
+
+
+LayoutLMForSequenceClassification,pass,7
+
+
+
+M2M100ForConditionalGeneration,pass,4
+
+
+
+MBartForCausalLM,pass,12
+
+
+
+MBartForConditionalGeneration,pass,24
+
+
+
+MT5ForConditionalGeneration,pass,5
+
+
+
+MegatronBertForCausalLM,pass,5
+
+
+
+MegatronBertForQuestionAnswering,pass,5
+
+
+
+MobileBertForMaskedLM,pass,3
+
+
+
+MobileBertForQuestionAnswering,pass,3
+
+
+
+OPTForCausalLM,pass,12
+
+
+
+PLBartForCausalLM,pass,12
+
+
+
+PLBartForConditionalGeneration,pass,29
+
+
+
+PegasusForCausalLM,pass,12
+
+
+
+PegasusForConditionalGeneration,pass,23
+
+
+
+RobertaForCausalLM,pass,5
+
+
+
+RobertaForQuestionAnswering,pass,5
+
+
+
+Speech2Text2ForCausalLM,pass,12
+
+
+
+T5ForConditionalGeneration,pass,5
+
+
+
+T5Small,pass,5
+
+
+
+TrOCRForCausalLM,pass,12
+
+
+
+XGLMForCausalLM,pass,12
+
+
+
+XLNetLMHeadModel,pass,5
+
+
+
+YituTechConvBert,pass,5

benchmarks/dynamo/ci_expected_accuracy/cu124/dynamic_inductor_timm_inference.csv

@@ -0,0 +1,245 @@
+name,accuracy,graph_breaks
+
+
+
+adv_inception_v3,pass,0
+
+
+
+beit_base_patch16_224,pass,0
+
+
+
+botnet26t_256,pass,0
+
+
+
+cait_m36_384,pass,0
+
+
+
+coat_lite_mini,pass,0
+
+
+
+convit_base,pass,0
+
+
+
+convmixer_768_32,pass,0
+
+
+
+convnext_base,pass,0
+
+
+
+crossvit_9_240,pass,0
+
+
+
+cspdarknet53,pass,0
+
+
+
+deit_base_distilled_patch16_224,pass,0
+
+
+
+dla102,pass,0
+
+
+
+dm_nfnet_f0,pass,0
+
+
+
+dpn107,pass,0
+
+
+
+eca_botnext26ts_256,pass,0
+
+
+
+eca_halonext26ts,pass,0
+
+
+
+ese_vovnet19b_dw,pass,0
+
+
+
+fbnetc_100,pass,0
+
+
+
+fbnetv3_b,pass,0
+
+
+
+gernet_l,pass,0
+
+
+
+ghostnet_100,pass,0
+
+
+
+gluon_inception_v3,pass,0
+
+
+
+gmixer_24_224,pass,0
+
+
+
+gmlp_s16_224,pass,0
+
+
+
+hrnet_w18,pass,0
+
+
+
+inception_v3,pass,0
+
+
+
+jx_nest_base,pass,0
+
+
+
+lcnet_050,pass,0
+
+
+
+levit_128,pass,0
+
+
+
+mixer_b16_224,pass,0
+
+
+
+mixnet_l,pass,0
+
+
+
+mnasnet_100,pass,0
+
+
+
+mobilenetv2_100,pass,0
+
+
+
+mobilenetv3_large_100,pass,0
+
+
+
+mobilevit_s,pass,0
+
+
+
+nfnet_l0,pass,0
+
+
+
+pit_b_224,pass,0
+
+
+
+pnasnet5large,pass,0
+
+
+
+poolformer_m36,pass,0
+
+
+
+regnety_002,pass,0
+
+
+
+repvgg_a2,pass,0
+
+
+
+res2net101_26w_4s,pass,0
+
+
+
+res2net50_14w_8s,pass,0
+
+
+
+res2next50,pass,0
+
+
+
+resmlp_12_224,pass,0
+
+
+
+resnest101e,pass,0
+
+
+
+rexnet_100,pass,0
+
+
+
+sebotnet33ts_256,pass,0
+
+
+
+selecsls42b,pass,0
+
+
+
+spnasnet_100,pass,0
+
+
+
+swin_base_patch4_window7_224,pass,0
+
+
+
+swsl_resnext101_32x16d,pass,0
+
+
+
+tf_efficientnet_b0,pass,0
+
+
+
+tf_mixnet_l,pass,0
+
+
+
+tinynet_a,pass,0
+
+
+
+tnt_s_patch16_224,pass,0
+
+
+
+twins_pcpvt_base,pass,0
+
+
+
+visformer_small,pass,0
+
+
+
+vit_base_patch16_224,pass,0
+
+
+
+volo_d1_224,pass,0
+
+
+
+xcit_large_24_p8_224,pass,0

benchmarks/dynamo/ci_expected_accuracy/cu124/dynamic_inductor_timm_training.csv

@@ -0,0 +1,245 @@
+name,accuracy,graph_breaks
+
+
+
+adv_inception_v3,pass,6
+
+
+
+beit_base_patch16_224,fail_accuracy,7
+
+
+
+botnet26t_256,pass,6
+
+
+
+cait_m36_384,eager_fail_to_run,0
+
+
+
+coat_lite_mini,pass,6
+
+
+
+convit_base,pass,7
+
+
+
+convmixer_768_32,pass,5
+
+
+
+convnext_base,pass,7
+
+
+
+crossvit_9_240,pass,7
+
+
+
+cspdarknet53,fail_accuracy,7
+
+
+
+deit_base_distilled_patch16_224,pass,7
+
+
+
+dla102,pass,7
+
+
+
+dm_nfnet_f0,pass,6
+
+
+
+dpn107,pass,6
+
+
+
+eca_botnext26ts_256,pass,7
+
+
+
+eca_halonext26ts,pass,7
+
+
+
+ese_vovnet19b_dw,pass,7
+
+
+
+fbnetc_100,pass,7
+
+
+
+fbnetv3_b,pass,6
+
+
+
+gernet_l,pass,6
+
+
+
+ghostnet_100,pass,6
+
+
+
+gluon_inception_v3,fail_accuracy,7
+
+
+
+gmixer_24_224,pass,6
+
+
+
+gmlp_s16_224,pass,7
+
+
+
+hrnet_w18,pass,5
+
+
+
+inception_v3,pass,6
+
+
+
+jx_nest_base,pass,7
+
+
+
+lcnet_050,pass,6
+
+
+
+levit_128,pass,7
+
+
+
+mixer_b16_224,pass,7
+
+
+
+mixnet_l,pass,6
+
+
+
+mnasnet_100,pass,7
+
+
+
+mobilenetv2_100,pass,7
+
+
+
+mobilenetv3_large_100,pass,7
+
+
+
+mobilevit_s,pass,6
+
+
+
+nfnet_l0,pass,7
+
+
+
+pit_b_224,pass,6
+
+
+
+pnasnet5large,pass,5
+
+
+
+poolformer_m36,pass,6
+
+
+
+regnety_002,pass,6
+
+
+
+repvgg_a2,pass,7
+
+
+
+res2net101_26w_4s,pass,6
+
+
+
+res2net50_14w_8s,pass,6
+
+
+
+res2next50,pass,6
+
+
+
+resmlp_12_224,pass,6
+
+
+
+resnest101e,pass,6
+
+
+
+rexnet_100,pass,7
+
+
+
+sebotnet33ts_256,pass,6
+
+
+
+selecsls42b,pass,6
+
+
+
+spnasnet_100,pass,7
+
+
+
+swin_base_patch4_window7_224,pass,7
+
+
+
+swsl_resnext101_32x16d,pass,6
+
+
+
+tf_efficientnet_b0,pass,6
+
+
+
+tf_mixnet_l,pass,6
+
+
+
+tinynet_a,pass,6
+
+
+
+tnt_s_patch16_224,pass,7
+
+
+
+twins_pcpvt_base,pass,7
+
+
+
+visformer_small,pass,7
+
+
+
+vit_base_patch16_224,pass,7
+
+
+
+volo_d1_224,pass,7
+
+
+
+xcit_large_24_p8_224,pass_due_to_skip,7

benchmarks/dynamo/ci_expected_accuracy/cu124/dynamic_inductor_torchbench_inference.csv

@@ -0,0 +1,377 @@
+name,accuracy,graph_breaks
+
+
+
+torchrec_dlrm,eager_fail_to_run,0
+
+
+
+BERT_pytorch,pass,0
+
+
+
+Background_Matting,pass_due_to_skip,0
+
+
+
+DALLE2_pytorch,pass,12
+
+
+
+LearningToPaint,pass,0
+
+
+
+Super_SloMo,pass,0
+
+
+
+alexnet,pass,0
+
+
+
+basic_gnn_edgecnn,pass,0
+
+
+
+basic_gnn_gcn,pass,6
+
+
+
+basic_gnn_gin,pass,0
+
+
+
+basic_gnn_sage,pass,0
+
+
+
+cm3leon_generate,pass,4
+
+
+
+dcgan,pass,0
+
+
+
+demucs,pass,3
+
+
+
+densenet121,pass,0
+
+
+
+detectron2_fasterrcnn_r_101_c4,eager_fail_to_run,0
+
+
+
+detectron2_fasterrcnn_r_101_dc5,eager_fail_to_run,0
+
+
+
+detectron2_fasterrcnn_r_101_fpn,eager_fail_to_run,0
+
+
+
+detectron2_fasterrcnn_r_50_c4,eager_fail_to_run,0
+
+
+
+detectron2_fasterrcnn_r_50_dc5,eager_fail_to_run,0
+
+
+
+detectron2_fasterrcnn_r_50_fpn,eager_fail_to_run,0
+
+
+
+detectron2_fcos_r_50_fpn,pass,21
+
+
+
+detectron2_maskrcnn_r_101_c4,eager_fail_to_run,0
+
+
+
+detectron2_maskrcnn_r_101_fpn,eager_fail_to_run,0
+
+
+
+detectron2_maskrcnn_r_50_c4,eager_fail_to_run,0
+
+
+
+detectron2_maskrcnn_r_50_fpn,eager_fail_to_run,0
+
+
+
+dlrm,pass,0
+
+
+
+doctr_det_predictor,pass,5
+
+
+
+doctr_reco_predictor,pass,4
+
+
+
+drq,pass,0
+
+
+
+fastNLP_Bert,pass,4
+
+
+
+functorch_dp_cifar10,pass,0
+
+
+
+functorch_maml_omniglot,pass,0
+
+
+
+hf_Albert,pass,0
+
+
+
+hf_Bart,pass,0
+
+
+
+hf_Bert,pass,0
+
+
+
+hf_Bert_large,pass,0
+
+
+
+hf_BigBird,fail_accuracy,46
+
+
+
+hf_DistilBert,pass,0
+
+
+
+hf_GPT2,pass,0
+
+
+
+hf_GPT2_large,pass_due_to_skip,0
+
+
+
+hf_Reformer,pass,5
+
+
+
+hf_T5,pass,0
+
+
+
+hf_T5_base,eager_fail_to_run,0
+
+
+
+hf_T5_generate,fail_to_run,5
+
+
+
+hf_T5_large,pass_due_to_skip,0
+
+
+
+hf_Whisper,pass,0
+
+
+
+hf_distil_whisper,pass,0
+
+
+
+lennard_jones,pass,0
+
+
+
+llama,pass,0
+
+
+
+llama_v2_7b_16h,model_fail_to_load,0
+
+
+
+llava,model_fail_to_load,0
+
+
+
+maml,pass_due_to_skip,0
+
+
+
+maml_omniglot,pass,0
+
+
+
+mnasnet1_0,pass,0
+
+
+
+mobilenet_v2,pass,0
+
+
+
+mobilenet_v2_quantized_qat,model_fail_to_load,0
+
+
+
+mobilenet_v3_large,pass,0
+
+
+
+moco,pass,5
+
+
+
+moondream,model_fail_to_load,0
+
+
+
+nanogpt,pass,0
+
+
+
+nvidia_deeprecommender,pass,0
+
+
+
+opacus_cifar10,pass,0
+
+
+
+phlippe_densenet,pass,0
+
+
+
+phlippe_resnet,pass,0
+
+
+
+pyhpc_equation_of_state,pass,0
+
+
+
+pyhpc_isoneutral_mixing,pass,0
+
+
+
+pyhpc_turbulent_kinetic_energy,pass,0
+
+
+
+pytorch_CycleGAN_and_pix2pix,pass,0
+
+
+
+pytorch_stargan,pass,0
+
+
+
+pytorch_unet,pass,0
+
+
+
+resnet152,pass,0
+
+
+
+resnet18,pass,0
+
+
+
+resnet50,pass,0
+
+
+
+resnet50_quantized_qat,model_fail_to_load,0
+
+
+
+resnext50_32x4d,pass,0
+
+
+
+sam,pass,0
+
+
+
+shufflenet_v2_x1_0,pass,0
+
+
+
+soft_actor_critic,pass,0
+
+
+
+speech_transformer,pass,10
+
+
+
+squeezenet1_1,pass,0
+
+
+
+stable_diffusion_text_encoder,pass,0
+
+
+
+stable_diffusion_unet,pass_due_to_skip,0
+
+
+
+timm_efficientnet,pass,0
+
+
+
+timm_regnet,pass,0
+
+
+
+timm_resnest,pass,0
+
+
+
+timm_vision_transformer,pass,0
+
+
+
+timm_vision_transformer_large,pass_due_to_skip,0
+
+
+
+timm_vovnet,pass,0
+
+
+
+torch_multimodal_clip,pass,0
+
+
+
+tts_angular,pass,2
+
+
+
+vgg16,pass,0
+
+
+
+vision_maskrcnn,pass,17
+
+
+
+yolov3,pass,2

benchmarks/dynamo/ci_expected_accuracy/cu124/dynamic_inductor_torchbench_training.csv

@@ -0,0 +1,285 @@
+name,accuracy,graph_breaks
+
+
+
+torchrec_dlrm,fail_to_run,3
+
+
+
+BERT_pytorch,pass,6
+
+
+
+Background_Matting,pass_due_to_skip,0
+
+
+
+DALLE2_pytorch,eager_fail_to_run,0
+
+
+
+LearningToPaint,pass,6
+
+
+
+Super_SloMo,pass,7
+
+
+
+alexnet,pass,6
+
+
+
+basic_gnn_edgecnn,pass,22
+
+
+
+basic_gnn_gcn,pass,13
+
+
+
+basic_gnn_gin,pass,7
+
+
+
+basic_gnn_sage,pass,7
+
+
+
+dcgan,pass,6
+
+
+
+demucs,fail_to_run,4
+
+
+
+densenet121,pass,6
+
+
+
+detectron2_maskrcnn_r_50_c4,eager_fail_to_run,0
+
+
+
+dlrm,pass,6
+
+
+
+drq,pass,6
+
+
+
+fastNLP_Bert,pass,10
+
+
+
+functorch_dp_cifar10,pass,7
+
+
+
+functorch_maml_omniglot,pass,7
+
+
+
+hf_Albert,pass,6
+
+
+
+hf_Bart,pass,6
+
+
+
+hf_Bert,pass,6
+
+
+
+hf_Bert_large,pass,6
+
+
+
+hf_BigBird,pass,52
+
+
+
+hf_DistilBert,pass,6
+
+
+
+hf_GPT2,pass,6
+
+
+
+hf_GPT2_large,pass_due_to_skip,0
+
+
+
+hf_Reformer,pass,26
+
+
+
+hf_T5_base,eager_2nd_run_OOM,0
+
+
+
+hf_T5_large,pass_due_to_skip,0
+
+
+
+hf_Whisper,pass,6
+
+
+
+hf_distil_whisper,model_fail_to_load,0
+
+
+
+lennard_jones,pass,7
+
+
+
+llava,model_fail_to_load,0
+
+
+
+maml_omniglot,pass,7
+
+
+
+mnasnet1_0,pass,7
+
+
+
+mobilenet_v2,pass,6
+
+
+
+mobilenet_v2_quantized_qat,eager_fail_to_run,0
+
+
+
+mobilenet_v3_large,pass,7
+
+
+
+moco,pass,11
+
+
+
+nanogpt,pass,7
+
+
+
+nvidia_deeprecommender,pass,7
+
+
+
+opacus_cifar10,eager_fail_to_run,0
+
+
+
+phlippe_densenet,pass,6
+
+
+
+phlippe_resnet,fail_accuracy,6
+
+
+
+pytorch_CycleGAN_and_pix2pix,pass,6
+
+
+
+pytorch_stargan,pass,6
+
+
+
+pytorch_unet,pass_due_to_skip,7
+
+
+
+resnet152,pass,7
+
+
+
+resnet18,pass,6
+
+
+
+resnet50,pass,6
+
+
+
+resnet50_quantized_qat,eager_fail_to_run,0
+
+
+
+resnext50_32x4d,pass,7
+
+
+
+sam,eager_fail_to_run,0
+
+
+
+shufflenet_v2_x1_0,pass,6
+
+
+
+soft_actor_critic,pass,6
+
+
+
+squeezenet1_1,pass,6
+
+
+
+stable_diffusion_text_encoder,pass,5
+
+
+
+stable_diffusion_unet,pass_due_to_skip,0
+
+
+
+timm_efficientnet,pass,7
+
+
+
+timm_regnet,pass,6
+
+
+
+timm_resnest,pass,7
+
+
+
+timm_vision_transformer,pass,6
+
+
+
+timm_vision_transformer_large,pass_due_to_skip,0
+
+
+
+timm_vovnet,pass,6
+
+
+
+torch_multimodal_clip,pass,7
+
+
+
+tts_angular,pass,9
+
+
+
+vgg16,pass,6
+
+
+
+vision_maskrcnn,pass,34
+
+
+
+yolov3,pass,9

benchmarks/dynamo/ci_expected_accuracy/cu124/dynamo_eager_huggingface_inference.csv

@@ -0,0 +1,185 @@
+name,accuracy,graph_breaks
+
+
+
+AlbertForMaskedLM,pass,0
+
+
+
+AlbertForQuestionAnswering,pass,0
+
+
+
+AllenaiLongformerBase,pass,4
+
+
+
+BartForCausalLM,pass,0
+
+
+
+BartForConditionalGeneration,pass,0
+
+
+
+BertForMaskedLM,pass,0
+
+
+
+BertForQuestionAnswering,pass,0
+
+
+
+BlenderbotForCausalLM,pass_due_to_skip,0
+
+
+
+BlenderbotSmallForCausalLM,pass,0
+
+
+
+BlenderbotSmallForConditionalGeneration,pass,0
+
+
+
+CamemBert,pass,0
+
+
+
+DebertaForMaskedLM,pass,0
+
+
+
+DebertaForQuestionAnswering,pass,0
+
+
+
+DebertaV2ForMaskedLM,pass_due_to_skip,0
+
+
+
+DebertaV2ForQuestionAnswering,pass,0
+
+
+
+DistilBertForMaskedLM,pass,0
+
+
+
+DistilBertForQuestionAnswering,pass,0
+
+
+
+DistillGPT2,pass,0
+
+
+
+ElectraForCausalLM,pass,0
+
+
+
+ElectraForQuestionAnswering,pass,0
+
+
+
+GPT2ForSequenceClassification,pass,2
+
+
+
+GoogleFnet,pass,0
+
+
+
+LayoutLMForMaskedLM,pass,0
+
+
+
+LayoutLMForSequenceClassification,pass,2
+
+
+
+M2M100ForConditionalGeneration,pass,0
+
+
+
+MBartForCausalLM,pass,0
+
+
+
+MBartForConditionalGeneration,pass,0
+
+
+
+MT5ForConditionalGeneration,pass,0
+
+
+
+MegatronBertForCausalLM,pass,0
+
+
+
+MegatronBertForQuestionAnswering,pass,0
+
+
+
+MobileBertForMaskedLM,pass,0
+
+
+
+MobileBertForQuestionAnswering,pass,0
+
+
+
+OPTForCausalLM,pass,0
+
+
+
+PLBartForCausalLM,pass,0
+
+
+
+PLBartForConditionalGeneration,pass,0
+
+
+
+PegasusForCausalLM,pass,0
+
+
+
+PegasusForConditionalGeneration,pass,0
+
+
+
+RobertaForCausalLM,pass,0
+
+
+
+RobertaForQuestionAnswering,pass,0
+
+
+
+Speech2Text2ForCausalLM,pass,0
+
+
+
+T5ForConditionalGeneration,pass,0
+
+
+
+T5Small,pass,0
+
+
+
+TrOCRForCausalLM,pass,0
+
+
+
+XGLMForCausalLM,pass,0
+
+
+
+XLNetLMHeadModel,pass,0
+
+
+
+YituTechConvBert,pass,0

benchmarks/dynamo/ci_expected_accuracy/cu124/dynamo_eager_huggingface_training.csv

@@ -0,0 +1,185 @@
+name,accuracy,graph_breaks
+
+
+
+AlbertForMaskedLM,pass,4
+
+
+
+AlbertForQuestionAnswering,pass,5
+
+
+
+AllenaiLongformerBase,pass,9
+
+
+
+BartForCausalLM,pass,12
+
+
+
+BartForConditionalGeneration,pass,24
+
+
+
+BertForMaskedLM,pass,5
+
+
+
+BertForQuestionAnswering,pass,5
+
+
+
+BlenderbotForCausalLM,eager_fail_to_run,0
+
+
+
+BlenderbotSmallForCausalLM,pass,12
+
+
+
+BlenderbotSmallForConditionalGeneration,pass,24
+
+
+
+CamemBert,pass,5
+
+
+
+DebertaForMaskedLM,pass,5
+
+
+
+DebertaForQuestionAnswering,pass,5
+
+
+
+DebertaV2ForMaskedLM,pass_due_to_skip,0
+
+
+
+DebertaV2ForQuestionAnswering,eager_1st_run_OOM,0
+
+
+
+DistilBertForMaskedLM,pass,5
+
+
+
+DistilBertForQuestionAnswering,pass,5
+
+
+
+DistillGPT2,pass,5
+
+
+
+ElectraForCausalLM,pass,4
+
+
+
+ElectraForQuestionAnswering,pass,5
+
+
+
+GPT2ForSequenceClassification,pass,7
+
+
+
+GoogleFnet,pass,5
+
+
+
+LayoutLMForMaskedLM,pass,5
+
+
+
+LayoutLMForSequenceClassification,pass,7
+
+
+
+M2M100ForConditionalGeneration,pass,4
+
+
+
+MBartForCausalLM,pass,12
+
+
+
+MBartForConditionalGeneration,pass,24
+
+
+
+MT5ForConditionalGeneration,pass,5
+
+
+
+MegatronBertForCausalLM,pass,5
+
+
+
+MegatronBertForQuestionAnswering,pass,5
+
+
+
+MobileBertForMaskedLM,pass,3
+
+
+
+MobileBertForQuestionAnswering,pass,3
+
+
+
+OPTForCausalLM,pass,12
+
+
+
+PLBartForCausalLM,pass,12
+
+
+
+PLBartForConditionalGeneration,pass,29
+
+
+
+PegasusForCausalLM,pass,12
+
+
+
+PegasusForConditionalGeneration,pass,23
+
+
+
+RobertaForCausalLM,pass,5
+
+
+
+RobertaForQuestionAnswering,pass,5
+
+
+
+Speech2Text2ForCausalLM,pass,12
+
+
+
+T5ForConditionalGeneration,pass,5
+
+
+
+T5Small,pass,5
+
+
+
+TrOCRForCausalLM,pass,12
+
+
+
+XGLMForCausalLM,pass,12
+
+
+
+XLNetLMHeadModel,pass,5
+
+
+
+YituTechConvBert,pass,5

benchmarks/dynamo/ci_expected_accuracy/cu124/dynamo_eager_timm_inference.csv

@@ -0,0 +1,245 @@
+name,accuracy,graph_breaks
+
+
+
+adv_inception_v3,pass,0
+
+
+
+beit_base_patch16_224,pass,0
+
+
+
+botnet26t_256,pass,0
+
+
+
+cait_m36_384,pass,0
+
+
+
+coat_lite_mini,pass,0
+
+
+
+convit_base,pass,0
+
+
+
+convmixer_768_32,pass,0
+
+
+
+convnext_base,pass,0
+
+
+
+crossvit_9_240,pass,0
+
+
+
+cspdarknet53,pass,0
+
+
+
+deit_base_distilled_patch16_224,pass,0
+
+
+
+dla102,pass,0
+
+
+
+dm_nfnet_f0,pass,0
+
+
+
+dpn107,pass,0
+
+
+
+eca_botnext26ts_256,pass,0
+
+
+
+eca_halonext26ts,pass,0
+
+
+
+ese_vovnet19b_dw,pass,0
+
+
+
+fbnetc_100,pass,0
+
+
+
+fbnetv3_b,pass,0
+
+
+
+gernet_l,pass,0
+
+
+
+ghostnet_100,pass,0
+
+
+
+gluon_inception_v3,pass,0
+
+
+
+gmixer_24_224,pass,0
+
+
+
+gmlp_s16_224,pass,0
+
+
+
+hrnet_w18,pass,0
+
+
+
+inception_v3,pass,0
+
+
+
+jx_nest_base,pass,0
+
+
+
+lcnet_050,pass,0
+
+
+
+levit_128,pass,0
+
+
+
+mixer_b16_224,pass,0
+
+
+
+mixnet_l,pass,0
+
+
+
+mnasnet_100,pass,0
+
+
+
+mobilenetv2_100,pass,0
+
+
+
+mobilenetv3_large_100,pass,0
+
+
+
+mobilevit_s,pass,0
+
+
+
+nfnet_l0,pass,0
+
+
+
+pit_b_224,pass,0
+
+
+
+pnasnet5large,pass,0
+
+
+
+poolformer_m36,pass,0
+
+
+
+regnety_002,pass,0
+
+
+
+repvgg_a2,pass,0
+
+
+
+res2net101_26w_4s,pass,0
+
+
+
+res2net50_14w_8s,pass,0
+
+
+
+res2next50,pass,0
+
+
+
+resmlp_12_224,pass,0
+
+
+
+resnest101e,pass,0
+
+
+
+rexnet_100,pass,0
+
+
+
+sebotnet33ts_256,pass,0
+
+
+
+selecsls42b,pass,0
+
+
+
+spnasnet_100,pass,0
+
+
+
+swin_base_patch4_window7_224,pass,0
+
+
+
+swsl_resnext101_32x16d,pass,0
+
+
+
+tf_efficientnet_b0,pass,0
+
+
+
+tf_mixnet_l,pass,0
+
+
+
+tinynet_a,pass,0
+
+
+
+tnt_s_patch16_224,pass,0
+
+
+
+twins_pcpvt_base,pass,0
+
+
+
+visformer_small,pass,0
+
+
+
+vit_base_patch16_224,pass,0
+
+
+
+volo_d1_224,pass,0
+
+
+
+xcit_large_24_p8_224,pass,0

benchmarks/dynamo/ci_expected_accuracy/cu124/dynamo_eager_timm_training.csv

@@ -0,0 +1,245 @@
+name,accuracy,graph_breaks
+
+
+
+adv_inception_v3,pass,6
+
+
+
+beit_base_patch16_224,pass,7
+
+
+
+botnet26t_256,pass,6
+
+
+
+cait_m36_384,eager_fail_to_run,0
+
+
+
+coat_lite_mini,pass,6
+
+
+
+convit_base,pass,7
+
+
+
+convmixer_768_32,pass,5
+
+
+
+convnext_base,pass,7
+
+
+
+crossvit_9_240,pass,7
+
+
+
+cspdarknet53,pass,7
+
+
+
+deit_base_distilled_patch16_224,pass,7
+
+
+
+dla102,pass,7
+
+
+
+dm_nfnet_f0,pass,6
+
+
+
+dpn107,pass,6
+
+
+
+eca_botnext26ts_256,pass,7
+
+
+
+eca_halonext26ts,pass,7
+
+
+
+ese_vovnet19b_dw,pass,7
+
+
+
+fbnetc_100,pass,7
+
+
+
+fbnetv3_b,pass,6
+
+
+
+gernet_l,pass,6
+
+
+
+ghostnet_100,pass,6
+
+
+
+gluon_inception_v3,pass,7
+
+
+
+gmixer_24_224,pass,6
+
+
+
+gmlp_s16_224,pass,7
+
+
+
+hrnet_w18,pass,5
+
+
+
+inception_v3,pass,6
+
+
+
+jx_nest_base,pass,7
+
+
+
+lcnet_050,pass,6
+
+
+
+levit_128,pass,7
+
+
+
+mixer_b16_224,pass,7
+
+
+
+mixnet_l,pass,6
+
+
+
+mnasnet_100,pass,7
+
+
+
+mobilenetv2_100,pass,7
+
+
+
+mobilenetv3_large_100,pass,7
+
+
+
+mobilevit_s,pass,6
+
+
+
+nfnet_l0,pass,7
+
+
+
+pit_b_224,pass,6
+
+
+
+pnasnet5large,pass,5
+
+
+
+poolformer_m36,pass,6
+
+
+
+regnety_002,pass,6
+
+
+
+repvgg_a2,pass,7
+
+
+
+res2net101_26w_4s,pass,6
+
+
+
+res2net50_14w_8s,pass,6
+
+
+
+res2next50,pass,6
+
+
+
+resmlp_12_224,pass,6
+
+
+
+resnest101e,pass,6
+
+
+
+rexnet_100,pass,7
+
+
+
+sebotnet33ts_256,pass,6
+
+
+
+selecsls42b,pass,6
+
+
+
+spnasnet_100,pass,7
+
+
+
+swin_base_patch4_window7_224,pass,7
+
+
+
+swsl_resnext101_32x16d,pass,6
+
+
+
+tf_efficientnet_b0,pass,6
+
+
+
+tf_mixnet_l,pass,6
+
+
+
+tinynet_a,pass,6
+
+
+
+tnt_s_patch16_224,pass,7
+
+
+
+twins_pcpvt_base,pass,7
+
+
+
+visformer_small,pass,7
+
+
+
+vit_base_patch16_224,pass,7
+
+
+
+volo_d1_224,pass,7
+
+
+
+xcit_large_24_p8_224,pass_due_to_skip,7

benchmarks/dynamo/ci_expected_accuracy/cu124/dynamo_eager_torchbench_inference.csv

@@ -0,0 +1,381 @@
+name,accuracy,graph_breaks
+
+
+
+torchrec_dlrm,eager_fail_to_run,0
+
+
+
+BERT_pytorch,pass,0
+
+
+
+Background_Matting,pass_due_to_skip,0
+
+
+
+DALLE2_pytorch,pass,12
+
+
+
+LearningToPaint,pass,0
+
+
+
+Super_SloMo,pass,0
+
+
+
+alexnet,pass,0
+
+
+
+basic_gnn_edgecnn,pass,0
+
+
+
+basic_gnn_gcn,pass,6
+
+
+
+basic_gnn_gin,pass,0
+
+
+
+basic_gnn_sage,pass,0
+
+
+
+cm3leon_generate,pass,4
+
+
+
+dcgan,pass,0
+
+
+
+demucs,pass,3
+
+
+
+densenet121,pass,0
+
+
+
+detectron2_fasterrcnn_r_101_c4,eager_fail_to_run,0
+
+
+
+detectron2_fasterrcnn_r_101_dc5,eager_fail_to_run,0
+
+
+
+detectron2_fasterrcnn_r_101_fpn,eager_fail_to_run,0
+
+
+
+detectron2_fasterrcnn_r_50_c4,eager_fail_to_run,0
+
+
+
+detectron2_fasterrcnn_r_50_dc5,eager_fail_to_run,0
+
+
+
+detectron2_fasterrcnn_r_50_fpn,eager_fail_to_run,0
+
+
+
+detectron2_fcos_r_50_fpn,pass,21
+
+
+
+detectron2_maskrcnn_r_101_c4,eager_fail_to_run,0
+
+
+
+detectron2_maskrcnn_r_101_fpn,eager_fail_to_run,0
+
+
+
+detectron2_maskrcnn_r_50_c4,eager_fail_to_run,0
+
+
+
+detectron2_maskrcnn_r_50_fpn,eager_fail_to_run,0
+
+
+
+dlrm,pass,0
+
+
+
+doctr_det_predictor,pass,5
+
+
+
+doctr_reco_predictor,pass,4
+
+
+
+drq,pass,0
+
+
+
+fastNLP_Bert,pass,4
+
+
+
+functorch_dp_cifar10,pass,0
+
+
+
+functorch_maml_omniglot,pass,0
+
+
+
+hf_Albert,pass,0
+
+
+
+hf_Bart,pass,0
+
+
+
+hf_Bert,pass,0
+
+
+
+hf_Bert_large,pass,0
+
+
+
+hf_BigBird,pass,46
+
+
+
+hf_DistilBert,pass,0
+
+
+
+hf_GPT2,pass,0
+
+
+
+hf_GPT2_large,pass_due_to_skip,0
+
+
+
+hf_Reformer,pass,5
+
+
+
+hf_T5,pass,0
+
+
+
+hf_T5_base,eager_fail_to_run,0
+
+
+
+hf_T5_generate,pass,5
+
+
+
+hf_T5_large,pass_due_to_skip,0
+
+
+
+hf_Whisper,pass,0
+
+
+
+hf_distil_whisper,pass,0
+
+
+
+lennard_jones,pass,0
+
+
+
+llama,pass,0
+
+
+
+llama_v2_7b_16h,model_fail_to_load,0
+
+
+
+llava,model_fail_to_load,0
+
+
+
+maml,pass_due_to_skip,0
+
+
+
+maml_omniglot,pass,0
+
+
+
+mnasnet1_0,pass,0
+
+
+
+mobilenet_v2,pass,0
+
+
+
+mobilenet_v2_quantized_qat,model_fail_to_load,0
+
+
+
+mobilenet_v3_large,pass,0
+
+
+
+moco,pass,5
+
+
+
+moondream,model_fail_to_load,0
+
+
+
+nanogpt,pass,0
+
+
+
+nvidia_deeprecommender,pass,0
+
+
+
+opacus_cifar10,pass,0
+
+
+
+phlippe_densenet,pass,0
+
+
+
+phlippe_resnet,pass,0
+
+
+
+pyhpc_equation_of_state,pass,0
+
+
+
+pyhpc_isoneutral_mixing,pass,0
+
+
+
+pyhpc_turbulent_kinetic_energy,pass,0
+
+
+
+pytorch_CycleGAN_and_pix2pix,pass,0
+
+
+
+pytorch_stargan,pass,0
+
+
+
+pytorch_unet,pass,0
+
+
+
+resnet152,pass,0
+
+
+
+resnet18,pass,0
+
+
+
+resnet50,pass,0
+
+
+
+resnet50_quantized_qat,model_fail_to_load,0
+
+
+
+resnext50_32x4d,pass,0
+
+
+
+sam,pass,0
+
+
+
+sam_fast,pass,0
+
+
+
+shufflenet_v2_x1_0,pass,0
+
+
+
+soft_actor_critic,pass,0
+
+
+
+speech_transformer,pass,10
+
+
+
+squeezenet1_1,pass,0
+
+
+
+stable_diffusion_text_encoder,pass,0
+
+
+
+stable_diffusion_unet,pass_due_to_skip,0
+
+
+
+timm_efficientnet,pass,0
+
+
+
+timm_regnet,pass,0
+
+
+
+timm_resnest,pass,0
+
+
+
+timm_vision_transformer,pass,0
+
+
+
+timm_vision_transformer_large,pass_due_to_skip,0
+
+
+
+timm_vovnet,pass,0
+
+
+
+torch_multimodal_clip,pass,0
+
+
+
+tts_angular,pass,2
+
+
+
+vgg16,pass,0
+
+
+
+vision_maskrcnn,pass,17
+
+
+
+yolov3,pass,2

benchmarks/dynamo/ci_expected_accuracy/cu124/dynamo_eager_torchbench_training.csv

@@ -0,0 +1,289 @@
+name,accuracy,graph_breaks
+
+
+
+torchrec_dlrm,pass,6
+
+
+
+BERT_pytorch,pass,6
+
+
+
+Background_Matting,pass_due_to_skip,0
+
+
+
+DALLE2_pytorch,eager_fail_to_run,0
+
+
+
+LearningToPaint,pass,6
+
+
+
+Super_SloMo,pass,7
+
+
+
+alexnet,pass,6
+
+
+
+basic_gnn_edgecnn,pass,22
+
+
+
+basic_gnn_gcn,pass,13
+
+
+
+basic_gnn_gin,pass,7
+
+
+
+basic_gnn_sage,pass,7
+
+
+
+dcgan,pass,6
+
+
+
+demucs,pass,9
+
+
+
+densenet121,pass,6
+
+
+
+detectron2_maskrcnn_r_50_c4,eager_fail_to_run,0
+
+
+
+dlrm,pass,6
+
+
+
+drq,pass,6
+
+
+
+fastNLP_Bert,pass,10
+
+
+
+functorch_dp_cifar10,pass,7
+
+
+
+functorch_maml_omniglot,pass,7
+
+
+
+hf_Albert,pass,6
+
+
+
+hf_Bart,pass,6
+
+
+
+hf_Bert,pass,6
+
+
+
+hf_Bert_large,pass,6
+
+
+
+hf_BigBird,pass,52
+
+
+
+hf_DistilBert,pass,6
+
+
+
+hf_GPT2,pass,6
+
+
+
+hf_GPT2_large,pass_due_to_skip,0
+
+
+
+hf_Reformer,pass,26
+
+
+
+hf_T5_base,eager_2nd_run_OOM,0
+
+
+
+hf_T5_large,pass_due_to_skip,0
+
+
+
+hf_Whisper,pass,6
+
+
+
+hf_distil_whisper,model_fail_to_load,0
+
+
+
+lennard_jones,pass,7
+
+
+
+llava,model_fail_to_load,0
+
+
+
+maml_omniglot,pass,7
+
+
+
+mnasnet1_0,pass,7
+
+
+
+mobilenet_v2,pass,6
+
+
+
+mobilenet_v2_quantized_qat,eager_fail_to_run,0
+
+
+
+mobilenet_v3_large,pass,7
+
+
+
+moco,pass,11
+
+
+
+nanogpt,pass,7
+
+
+
+nvidia_deeprecommender,pass,7
+
+
+
+opacus_cifar10,eager_fail_to_run,0
+
+
+
+phlippe_densenet,pass,6
+
+
+
+phlippe_resnet,pass,6
+
+
+
+pytorch_CycleGAN_and_pix2pix,pass,6
+
+
+
+pytorch_stargan,pass,6
+
+
+
+pytorch_unet,pass_due_to_skip,7
+
+
+
+resnet152,pass,7
+
+
+
+resnet18,pass,6
+
+
+
+resnet50,pass,6
+
+
+
+resnet50_quantized_qat,eager_fail_to_run,0
+
+
+
+resnext50_32x4d,pass,7
+
+
+
+sam,eager_fail_to_run,0
+
+
+
+shufflenet_v2_x1_0,pass,6
+
+
+
+soft_actor_critic,pass,6
+
+
+
+speech_transformer,pass,16
+
+
+
+squeezenet1_1,pass,6
+
+
+
+stable_diffusion_text_encoder,pass,5
+
+
+
+stable_diffusion_unet,pass_due_to_skip,0
+
+
+
+timm_efficientnet,pass,7
+
+
+
+timm_regnet,pass,6
+
+
+
+timm_resnest,pass,7
+
+
+
+timm_vision_transformer,pass,6
+
+
+
+timm_vision_transformer_large,pass_due_to_skip,0
+
+
+
+timm_vovnet,pass,6
+
+
+
+torch_multimodal_clip,pass,7
+
+
+
+tts_angular,pass,9
+
+
+
+vgg16,pass,6
+
+
+
+vision_maskrcnn,pass,34
+
+
+
+yolov3,pass,9

benchmarks/dynamo/ci_expected_accuracy/cu124/inductor_huggingface_inference.csv

@@ -0,0 +1,185 @@
+name,accuracy,graph_breaks
+
+
+
+AlbertForMaskedLM,pass,0
+
+
+
+AlbertForQuestionAnswering,pass,0
+
+
+
+AllenaiLongformerBase,pass,4
+
+
+
+BartForCausalLM,pass,0
+
+
+
+BartForConditionalGeneration,pass,0
+
+
+
+BertForMaskedLM,pass,0
+
+
+
+BertForQuestionAnswering,pass,0
+
+
+
+BlenderbotForCausalLM,pass_due_to_skip,0
+
+
+
+BlenderbotSmallForCausalLM,pass,0
+
+
+
+BlenderbotSmallForConditionalGeneration,pass,0
+
+
+
+CamemBert,pass,0
+
+
+
+DebertaForMaskedLM,pass,0
+
+
+
+DebertaForQuestionAnswering,pass,0
+
+
+
+DebertaV2ForMaskedLM,pass_due_to_skip,0
+
+
+
+DebertaV2ForQuestionAnswering,pass,0
+
+
+
+DistilBertForMaskedLM,pass,0
+
+
+
+DistilBertForQuestionAnswering,pass,0
+
+
+
+DistillGPT2,pass,0
+
+
+
+ElectraForCausalLM,pass,0
+
+
+
+ElectraForQuestionAnswering,pass,0
+
+
+
+GPT2ForSequenceClassification,pass,2
+
+
+
+GoogleFnet,pass,0
+
+
+
+LayoutLMForMaskedLM,pass,0
+
+
+
+LayoutLMForSequenceClassification,pass,2
+
+
+
+M2M100ForConditionalGeneration,pass,0
+
+
+
+MBartForCausalLM,pass,0
+
+
+
+MBartForConditionalGeneration,pass,0
+
+
+
+MT5ForConditionalGeneration,pass,0
+
+
+
+MegatronBertForCausalLM,pass,0
+
+
+
+MegatronBertForQuestionAnswering,pass,0
+
+
+
+MobileBertForMaskedLM,pass,0
+
+
+
+MobileBertForQuestionAnswering,pass,0
+
+
+
+OPTForCausalLM,pass,0
+
+
+
+PLBartForCausalLM,pass,0
+
+
+
+PLBartForConditionalGeneration,pass,0
+
+
+
+PegasusForCausalLM,pass,0
+
+
+
+PegasusForConditionalGeneration,pass,0
+
+
+
+RobertaForCausalLM,pass,0
+
+
+
+RobertaForQuestionAnswering,pass,0
+
+
+
+Speech2Text2ForCausalLM,pass,0
+
+
+
+T5ForConditionalGeneration,pass,0
+
+
+
+T5Small,pass,0
+
+
+
+TrOCRForCausalLM,pass,0
+
+
+
+XGLMForCausalLM,pass,0
+
+
+
+XLNetLMHeadModel,pass,0
+
+
+
+YituTechConvBert,pass,0

benchmarks/dynamo/ci_expected_accuracy/cu124/inductor_huggingface_training.csv

@@ -0,0 +1,185 @@
+name,accuracy,graph_breaks
+
+
+
+AlbertForMaskedLM,pass,4
+
+
+
+AlbertForQuestionAnswering,pass,5
+
+
+
+AllenaiLongformerBase,pass,9
+
+
+
+BartForCausalLM,pass,12
+
+
+
+BartForConditionalGeneration,pass,24
+
+
+
+BertForMaskedLM,pass,5
+
+
+
+BertForQuestionAnswering,pass,5
+
+
+
+BlenderbotForCausalLM,eager_fail_to_run,0
+
+
+
+BlenderbotSmallForCausalLM,pass,12
+
+
+
+BlenderbotSmallForConditionalGeneration,pass,24
+
+
+
+CamemBert,pass,5
+
+
+
+DebertaForMaskedLM,pass,5
+
+
+
+DebertaForQuestionAnswering,pass,5
+
+
+
+DebertaV2ForMaskedLM,pass_due_to_skip,0
+
+
+
+DebertaV2ForQuestionAnswering,eager_1st_run_OOM,0
+
+
+
+DistilBertForMaskedLM,pass,5
+
+
+
+DistilBertForQuestionAnswering,pass,5
+
+
+
+DistillGPT2,pass,5
+
+
+
+ElectraForCausalLM,pass,4
+
+
+
+ElectraForQuestionAnswering,pass,5
+
+
+
+GPT2ForSequenceClassification,pass,7
+
+
+
+GoogleFnet,pass,5
+
+
+
+LayoutLMForMaskedLM,pass,5
+
+
+
+LayoutLMForSequenceClassification,pass,7
+
+
+
+M2M100ForConditionalGeneration,pass,4
+
+
+
+MBartForCausalLM,pass,12
+
+
+
+MBartForConditionalGeneration,pass,24
+
+
+
+MT5ForConditionalGeneration,pass,5
+
+
+
+MegatronBertForCausalLM,pass,5
+
+
+
+MegatronBertForQuestionAnswering,pass,5
+
+
+
+MobileBertForMaskedLM,pass,3
+
+
+
+MobileBertForQuestionAnswering,pass,3
+
+
+
+OPTForCausalLM,pass,12
+
+
+
+PLBartForCausalLM,pass,12
+
+
+
+PLBartForConditionalGeneration,pass,29
+
+
+
+PegasusForCausalLM,pass,12
+
+
+
+PegasusForConditionalGeneration,pass,23
+
+
+
+RobertaForCausalLM,pass,5
+
+
+
+RobertaForQuestionAnswering,pass,5
+
+
+
+Speech2Text2ForCausalLM,pass,12
+
+
+
+T5ForConditionalGeneration,pass,5
+
+
+
+T5Small,pass,5
+
+
+
+TrOCRForCausalLM,pass,12
+
+
+
+XGLMForCausalLM,pass,12
+
+
+
+XLNetLMHeadModel,pass,5
+
+
+
+YituTechConvBert,pass,5

benchmarks/dynamo/ci_expected_accuracy/cu124/inductor_timm_inference.csv

@@ -0,0 +1,245 @@
+name,accuracy,graph_breaks
+
+
+
+adv_inception_v3,pass,0
+
+
+
+beit_base_patch16_224,pass,0
+
+
+
+botnet26t_256,pass,0
+
+
+
+cait_m36_384,pass,0
+
+
+
+coat_lite_mini,pass,0
+
+
+
+convit_base,pass,0
+
+
+
+convmixer_768_32,pass,0
+
+
+
+convnext_base,pass,0
+
+
+
+crossvit_9_240,pass,0
+
+
+
+cspdarknet53,pass,0
+
+
+
+deit_base_distilled_patch16_224,pass,0
+
+
+
+dla102,pass,0
+
+
+
+dm_nfnet_f0,pass,0
+
+
+
+dpn107,pass,0
+
+
+
+eca_botnext26ts_256,pass,0
+
+
+
+eca_halonext26ts,pass,0
+
+
+
+ese_vovnet19b_dw,pass,0
+
+
+
+fbnetc_100,pass,0
+
+
+
+fbnetv3_b,pass,0
+
+
+
+gernet_l,pass,0
+
+
+
+ghostnet_100,pass,0
+
+
+
+gluon_inception_v3,pass,0
+
+
+
+gmixer_24_224,pass,0
+
+
+
+gmlp_s16_224,pass,0
+
+
+
+hrnet_w18,pass,0
+
+
+
+inception_v3,pass,0
+
+
+
+jx_nest_base,pass,0
+
+
+
+lcnet_050,pass,0
+
+
+
+levit_128,pass,0
+
+
+
+mixer_b16_224,pass,0
+
+
+
+mixnet_l,pass,0
+
+
+
+mnasnet_100,pass,0
+
+
+
+mobilenetv2_100,pass,0
+
+
+
+mobilenetv3_large_100,pass,0
+
+
+
+mobilevit_s,pass,0
+
+
+
+nfnet_l0,pass,0
+
+
+
+pit_b_224,pass,0
+
+
+
+pnasnet5large,pass,0
+
+
+
+poolformer_m36,pass,0
+
+
+
+regnety_002,pass,0
+
+
+
+repvgg_a2,pass,0
+
+
+
+res2net101_26w_4s,pass,0
+
+
+
+res2net50_14w_8s,pass,0
+
+
+
+res2next50,pass,0
+
+
+
+resmlp_12_224,pass,0
+
+
+
+resnest101e,pass,0
+
+
+
+rexnet_100,pass,0
+
+
+
+sebotnet33ts_256,pass,0
+
+
+
+selecsls42b,pass,0
+
+
+
+spnasnet_100,pass,0
+
+
+
+swin_base_patch4_window7_224,pass,0
+
+
+
+swsl_resnext101_32x16d,pass,0
+
+
+
+tf_efficientnet_b0,pass,0
+
+
+
+tf_mixnet_l,pass,0
+
+
+
+tinynet_a,pass,0
+
+
+
+tnt_s_patch16_224,pass,0
+
+
+
+twins_pcpvt_base,pass,0
+
+
+
+visformer_small,pass,0
+
+
+
+vit_base_patch16_224,pass,0
+
+
+
+volo_d1_224,pass,0
+
+
+
+xcit_large_24_p8_224,pass,0

benchmarks/dynamo/ci_expected_accuracy/cu124/inductor_timm_training.csv

@@ -0,0 +1,245 @@
+name,accuracy,graph_breaks
+
+
+
+adv_inception_v3,pass,6
+
+
+
+beit_base_patch16_224,fail_accuracy,7
+
+
+
+botnet26t_256,pass,6
+
+
+
+cait_m36_384,eager_fail_to_run,0
+
+
+
+coat_lite_mini,pass,6
+
+
+
+convit_base,pass,7
+
+
+
+convmixer_768_32,pass,5
+
+
+
+convnext_base,pass,7
+
+
+
+crossvit_9_240,pass,7
+
+
+
+cspdarknet53,pass,7
+
+
+
+deit_base_distilled_patch16_224,pass,7
+
+
+
+dla102,pass,7
+
+
+
+dm_nfnet_f0,pass,6
+
+
+
+dpn107,pass,6
+
+
+
+eca_botnext26ts_256,pass,7
+
+
+
+eca_halonext26ts,pass,7
+
+
+
+ese_vovnet19b_dw,pass,7
+
+
+
+fbnetc_100,pass,7
+
+
+
+fbnetv3_b,pass,6
+
+
+
+gernet_l,pass,6
+
+
+
+ghostnet_100,pass,6
+
+
+
+gluon_inception_v3,pass,7
+
+
+
+gmixer_24_224,pass,6
+
+
+
+gmlp_s16_224,pass,7
+
+
+
+hrnet_w18,pass,5
+
+
+
+inception_v3,pass,6
+
+
+
+jx_nest_base,pass,7
+
+
+
+lcnet_050,pass,6
+
+
+
+levit_128,pass,7
+
+
+
+mixer_b16_224,pass,7
+
+
+
+mixnet_l,pass,6
+
+
+
+mnasnet_100,pass,7
+
+
+
+mobilenetv2_100,pass,7
+
+
+
+mobilenetv3_large_100,pass,7
+
+
+
+mobilevit_s,pass,6
+
+
+
+nfnet_l0,pass,7
+
+
+
+pit_b_224,pass,6
+
+
+
+pnasnet5large,pass,5
+
+
+
+poolformer_m36,pass,6
+
+
+
+regnety_002,pass,6
+
+
+
+repvgg_a2,pass,7
+
+
+
+res2net101_26w_4s,pass,6
+
+
+
+res2net50_14w_8s,pass,6
+
+
+
+res2next50,pass,6
+
+
+
+resmlp_12_224,pass,6
+
+
+
+resnest101e,pass,6
+
+
+
+rexnet_100,pass,7
+
+
+
+sebotnet33ts_256,pass,6
+
+
+
+selecsls42b,pass,6
+
+
+
+spnasnet_100,pass,7
+
+
+
+swin_base_patch4_window7_224,pass,7
+
+
+
+swsl_resnext101_32x16d,pass,6
+
+
+
+tf_efficientnet_b0,pass,6
+
+
+
+tf_mixnet_l,pass,6
+
+
+
+tinynet_a,pass,6
+
+
+
+tnt_s_patch16_224,pass,7
+
+
+
+twins_pcpvt_base,pass,7
+
+
+
+visformer_small,pass,7
+
+
+
+vit_base_patch16_224,pass,7
+
+
+
+volo_d1_224,pass,7
+
+
+
+xcit_large_24_p8_224,pass_due_to_skip,7

benchmarks/dynamo/ci_expected_accuracy/cu124/inductor_torchbench_inference.csv

@@ -0,0 +1,381 @@
+name,accuracy,graph_breaks
+
+
+
+torchrec_dlrm,eager_fail_to_run,0
+
+
+
+BERT_pytorch,pass,0
+
+
+
+Background_Matting,pass_due_to_skip,0
+
+
+
+DALLE2_pytorch,pass,12
+
+
+
+LearningToPaint,pass,0
+
+
+
+Super_SloMo,pass,0
+
+
+
+alexnet,pass,0
+
+
+
+basic_gnn_edgecnn,pass,0
+
+
+
+basic_gnn_gcn,pass,6
+
+
+
+basic_gnn_gin,pass,0
+
+
+
+basic_gnn_sage,pass,0
+
+
+
+cm3leon_generate,pass,4
+
+
+
+dcgan,pass,0
+
+
+
+demucs,pass,3
+
+
+
+densenet121,pass,0
+
+
+
+detectron2_fasterrcnn_r_101_c4,eager_fail_to_run,0
+
+
+
+detectron2_fasterrcnn_r_101_dc5,eager_fail_to_run,0
+
+
+
+detectron2_fasterrcnn_r_101_fpn,eager_fail_to_run,0
+
+
+
+detectron2_fasterrcnn_r_50_c4,eager_fail_to_run,0
+
+
+
+detectron2_fasterrcnn_r_50_dc5,eager_fail_to_run,0
+
+
+
+detectron2_fasterrcnn_r_50_fpn,eager_fail_to_run,0
+
+
+
+detectron2_fcos_r_50_fpn,pass,21
+
+
+
+detectron2_maskrcnn_r_101_c4,eager_fail_to_run,0
+
+
+
+detectron2_maskrcnn_r_101_fpn,eager_fail_to_run,0
+
+
+
+detectron2_maskrcnn_r_50_c4,eager_fail_to_run,0
+
+
+
+detectron2_maskrcnn_r_50_fpn,eager_fail_to_run,0
+
+
+
+dlrm,pass,0
+
+
+
+doctr_det_predictor,pass,5
+
+
+
+doctr_reco_predictor,pass,4
+
+
+
+drq,pass,0
+
+
+
+fastNLP_Bert,pass,4
+
+
+
+functorch_dp_cifar10,pass,0
+
+
+
+functorch_maml_omniglot,pass,0
+
+
+
+hf_Albert,pass,0
+
+
+
+hf_Bart,pass,0
+
+
+
+hf_Bert,pass,0
+
+
+
+hf_Bert_large,pass,0
+
+
+
+hf_BigBird,fail_accuracy,46
+
+
+
+hf_DistilBert,pass,0
+
+
+
+hf_GPT2,pass,0
+
+
+
+hf_GPT2_large,pass_due_to_skip,0
+
+
+
+hf_Reformer,pass,5
+
+
+
+hf_T5,pass,0
+
+
+
+hf_T5_base,eager_fail_to_run,0
+
+
+
+hf_T5_generate,pass,5
+
+
+
+hf_T5_large,pass_due_to_skip,0
+
+
+
+hf_Whisper,pass,0
+
+
+
+hf_distil_whisper,pass,0
+
+
+
+lennard_jones,pass,0
+
+
+
+llama,pass,0
+
+
+
+llama_v2_7b_16h,model_fail_to_load,0
+
+
+
+llava,model_fail_to_load,0
+
+
+
+maml,pass_due_to_skip,0
+
+
+
+maml_omniglot,pass,0
+
+
+
+mnasnet1_0,pass,0
+
+
+
+mobilenet_v2,pass,0
+
+
+
+mobilenet_v2_quantized_qat,model_fail_to_load,0
+
+
+
+mobilenet_v3_large,pass,0
+
+
+
+moco,pass,5
+
+
+
+moondream,model_fail_to_load,0
+
+
+
+nanogpt,pass,0
+
+
+
+nvidia_deeprecommender,pass,0
+
+
+
+opacus_cifar10,pass,0
+
+
+
+phlippe_densenet,pass,0
+
+
+
+phlippe_resnet,pass,0
+
+
+
+pyhpc_equation_of_state,pass,0
+
+
+
+pyhpc_isoneutral_mixing,pass,0
+
+
+
+pyhpc_turbulent_kinetic_energy,pass,0
+
+
+
+pytorch_CycleGAN_and_pix2pix,pass,0
+
+
+
+pytorch_stargan,pass,0
+
+
+
+pytorch_unet,pass,0
+
+
+
+resnet152,pass,0
+
+
+
+resnet18,pass,0
+
+
+
+resnet50,pass,0
+
+
+
+resnet50_quantized_qat,model_fail_to_load,0
+
+
+
+resnext50_32x4d,pass,0
+
+
+
+sam,pass,0
+
+
+
+sam_fast,pass,0
+
+
+
+shufflenet_v2_x1_0,pass,0
+
+
+
+soft_actor_critic,pass,0
+
+
+
+speech_transformer,pass,10
+
+
+
+squeezenet1_1,pass,0
+
+
+
+stable_diffusion_text_encoder,pass,0
+
+
+
+stable_diffusion_unet,pass_due_to_skip,0
+
+
+
+timm_efficientnet,pass,0
+
+
+
+timm_regnet,pass,0
+
+
+
+timm_resnest,pass,0
+
+
+
+timm_vision_transformer,pass,0
+
+
+
+timm_vision_transformer_large,pass_due_to_skip,0
+
+
+
+timm_vovnet,pass,0
+
+
+
+torch_multimodal_clip,pass,0
+
+
+
+tts_angular,pass,2
+
+
+
+vgg16,pass,0
+
+
+
+vision_maskrcnn,pass,17
+
+
+
+yolov3,pass,2

benchmarks/dynamo/ci_expected_accuracy/cu124/inductor_torchbench_training.csv

@@ -0,0 +1,289 @@
+name,accuracy,graph_breaks
+
+
+
+torchrec_dlrm,pass,6
+
+
+
+BERT_pytorch,pass,6
+
+
+
+Background_Matting,pass_due_to_skip,0
+
+
+
+DALLE2_pytorch,eager_fail_to_run,0
+
+
+
+LearningToPaint,pass,6
+
+
+
+Super_SloMo,pass,7
+
+
+
+alexnet,pass,6
+
+
+
+basic_gnn_edgecnn,pass,22
+
+
+
+basic_gnn_gcn,pass,13
+
+
+
+basic_gnn_gin,pass,7
+
+
+
+basic_gnn_sage,pass,7
+
+
+
+dcgan,pass,6
+
+
+
+demucs,pass,9
+
+
+
+densenet121,pass,6
+
+
+
+detectron2_maskrcnn_r_50_c4,eager_fail_to_run,0
+
+
+
+dlrm,pass,6
+
+
+
+drq,pass,6
+
+
+
+fastNLP_Bert,pass,10
+
+
+
+functorch_dp_cifar10,pass,7
+
+
+
+functorch_maml_omniglot,pass,7
+
+
+
+hf_Albert,pass,6
+
+
+
+hf_Bart,pass,6
+
+
+
+hf_Bert,pass,6
+
+
+
+hf_Bert_large,pass,6
+
+
+
+hf_BigBird,pass,52
+
+
+
+hf_DistilBert,pass,6
+
+
+
+hf_GPT2,pass,6
+
+
+
+hf_GPT2_large,pass_due_to_skip,0
+
+
+
+hf_Reformer,pass,26
+
+
+
+hf_T5_base,eager_2nd_run_OOM,0
+
+
+
+hf_T5_large,pass_due_to_skip,0
+
+
+
+hf_Whisper,pass,6
+
+
+
+hf_distil_whisper,model_fail_to_load,0
+
+
+
+lennard_jones,pass,7
+
+
+
+llava,model_fail_to_load,0
+
+
+
+maml_omniglot,pass,7
+
+
+
+mnasnet1_0,pass,7
+
+
+
+mobilenet_v2,pass,6
+
+
+
+mobilenet_v2_quantized_qat,eager_fail_to_run,0
+
+
+
+mobilenet_v3_large,pass,7
+
+
+
+moco,pass,11
+
+
+
+nanogpt,pass,7
+
+
+
+nvidia_deeprecommender,pass,7
+
+
+
+opacus_cifar10,eager_fail_to_run,0
+
+
+
+phlippe_densenet,pass,6
+
+
+
+phlippe_resnet,fail_accuracy,6
+
+
+
+pytorch_CycleGAN_and_pix2pix,pass,6
+
+
+
+pytorch_stargan,pass,6
+
+
+
+pytorch_unet,pass_due_to_skip,7
+
+
+
+resnet152,pass,7
+
+
+
+resnet18,pass,6
+
+
+
+resnet50,pass,6
+
+
+
+resnet50_quantized_qat,eager_fail_to_run,0
+
+
+
+resnext50_32x4d,pass,7
+
+
+
+sam,eager_fail_to_run,0
+
+
+
+shufflenet_v2_x1_0,pass,6
+
+
+
+soft_actor_critic,pass,6
+
+
+
+speech_transformer,pass,16
+
+
+
+squeezenet1_1,pass,6
+
+
+
+stable_diffusion_text_encoder,pass,5
+
+
+
+stable_diffusion_unet,pass_due_to_skip,0
+
+
+
+timm_efficientnet,pass,7
+
+
+
+timm_regnet,pass,6
+
+
+
+timm_resnest,pass,7
+
+
+
+timm_vision_transformer,pass,6
+
+
+
+timm_vision_transformer_large,pass_due_to_skip,0
+
+
+
+timm_vovnet,pass,6
+
+
+
+torch_multimodal_clip,pass,7
+
+
+
+tts_angular,pass,9
+
+
+
+vgg16,pass,6
+
+
+
+vision_maskrcnn,pass,34
+
+
+
+yolov3,pass,9

benchmarks/dynamo/ci_expected_accuracy/cu124/update_expected.py

@@ -0,0 +1,172 @@
+"""
+Update commited CSV files used as reference points by dynamo/inductor CI.
+
+Currently only cares about graph breaks, so only saves those columns.
+
+Hardcodes a list of job names and artifacts per job, but builds the lookup
+by querying github sha and finding associated github actions workflow ID and CI jobs,
+downloading artifact zips, extracting CSVs and filtering them.
+
+Usage:
+
+python benchmarks/dynamo/ci_expected_accuracy.py <sha of pytorch commit that has completed inductor benchmark jobs>
+
+Known limitations:
+- doesn't handle 'retry' jobs in CI, if the same hash has more than one set of artifacts, gets the first one
+"""
+
+import argparse
+import json
+import os
+import pathlib
+import subprocess
+import sys
+import urllib
+from io import BytesIO
+from itertools import product
+from urllib.request import urlopen
+from zipfile import ZipFile
+
+import pandas as pd
+import requests
+
+# Note: the public query url targets this rockset lambda:
+# https://console.rockset.com/lambdas/details/commons.artifacts
+ARTIFACTS_QUERY_URL = "https://api.usw2a1.rockset.com/v1/public/shared_lambdas/4ca0033e-0117-41f5-b043-59cde19eff35"
+CSV_LINTER = str(
+    pathlib.Path(__file__).absolute().parent.parent.parent.parent
+    / "tools/linter/adapters/no_merge_conflict_csv_linter.py"
+)
+
+
+def query_job_sha(repo, sha):
+    params = {
+        "parameters": [
+            {"name": "sha", "type": "string", "value": sha},
+            {"name": "repo", "type": "string", "value": repo},
+        ]
+    }
+
+    r = requests.post(url=ARTIFACTS_QUERY_URL, json=params)
+    data = r.json()
+    return data["results"]
+
+
+def parse_job_name(job_str):
+    return (part.strip() for part in job_str.split("/"))
+
+
+def parse_test_str(test_str):
+    return (part.strip() for part in test_str[6:].strip(")").split(","))
+
+
+S3_BASE_URL = "https://gha-artifacts.s3.amazonaws.com"
+
+
+def get_artifacts_urls(results, suites):
+    urls = {}
+    for r in results:
+        if (
+            r["workflowName"] in ("inductor", "inductor-periodic")
+            and "test" in r["jobName"]
+        ):
+            config_str, test_str = parse_job_name(r["jobName"])
+            suite, shard_id, num_shards, machine, *_ = parse_test_str(test_str)
+            workflowId = r["workflowId"]
+            id = r["id"]
+            runAttempt = r["runAttempt"]
+
+            if suite in suites:
+                artifact_filename = f"test-reports-test-{suite}-{shard_id}-{num_shards}-{machine}_{id}.zip"
+                s3_url = f"{S3_BASE_URL}/{repo}/{workflowId}/{runAttempt}/artifact/{artifact_filename}"
+                urls[(suite, int(shard_id))] = s3_url
+                print(f"{suite} {shard_id}, {num_shards}: {s3_url}")
+    return urls
+
+
+def normalize_suite_filename(suite_name):
+    strs = suite_name.split("_")
+    subsuite = strs[-1]
+    if "timm" in subsuite:
+        subsuite = subsuite.replace("timm", "timm_models")
+
+    return subsuite
+
+
+def download_artifacts_and_extract_csvs(urls):
+    dataframes = {}
+    for (suite, shard), url in urls.items():
+        try:
+            resp = urlopen(url)
+            subsuite = normalize_suite_filename(suite)
+            artifact = ZipFile(BytesIO(resp.read()))
+            for phase in ("training", "inference"):
+                name = f"test/test-reports/{phase}_{subsuite}.csv"
+                try:
+                    df = pd.read_csv(artifact.open(name))
+                    df["graph_breaks"] = df["graph_breaks"].fillna(0).astype(int)
+                    prev_df = dataframes.get((suite, phase), None)
+                    dataframes[(suite, phase)] = (
+                        pd.concat([prev_df, df]) if prev_df is not None else df
+                    )
+                except KeyError:
+                    print(
+                        f"Warning: Unable to find {name} in artifacts file from {url}, continuing"
+                    )
+        except urllib.error.HTTPError:
+            print(f"Unable to download {url}, perhaps the CI job isn't finished?")
+
+    return dataframes
+
+
+def write_filtered_csvs(root_path, dataframes):
+    for (suite, phase), df in dataframes.items():
+        out_fn = os.path.join(root_path, f"{suite}_{phase}.csv")
+        df.to_csv(out_fn, index=False, columns=["name", "accuracy", "graph_breaks"])
+        apply_lints(out_fn)
+
+
+def apply_lints(filename):
+    patch = json.loads(subprocess.check_output([sys.executable, CSV_LINTER, filename]))
+    if patch.get("replacement"):
+        with open(filename) as fd:
+            data = fd.read().replace(patch["original"], patch["replacement"])
+        with open(filename, "w") as fd:
+            fd.write(data)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+
+    parser.add_argument("sha")
+    args = parser.parse_args()
+
+    repo = "pytorch/pytorch"
+
+    suites = {
+        f"{a}_{b}"
+        for a, b in product(
+            [
+                "aot_eager",
+                "aot_inductor",
+                "cpu_inductor",
+                "dynamic_aot_eager",
+                "dynamic_cpu_inductor",
+                "dynamic_inductor",
+                "dynamo_eager",
+                "inductor",
+            ],
+            ["huggingface", "timm", "torchbench"],
+        )
+    }
+
+    root_path = "benchmarks/dynamo/ci_expected_accuracy/"
+    assert os.path.exists(root_path), f"cd <pytorch root> and ensure {root_path} exists"
+
+    results = query_job_sha(repo, args.sha)
+    urls = get_artifacts_urls(results, suites)
+    dataframes = download_artifacts_and_extract_csvs(urls)
+    write_filtered_csvs(root_path, dataframes)
+    print("Success. Now, confirm the changes to .csvs and `git add` them if satisfied.")