diff --git a/java/build.gradle b/java/build.gradle
index cebf67e085..3219b08299 100644
--- a/java/build.gradle
+++ b/java/build.gradle
@@ -166,11 +166,14 @@ if (cmakeBuildDir != null) {
 	}
 
 	tasks.register('cmakeCheck', Copy) {
+		group = 'verification'
 		from layout.buildDirectory.get()
 		include 'reports/**'
 		into cmakeBuildOutputDir
 		dependsOn(check)
 	}
+} else {
+	println "cmakeBuildDir is not set. Skipping cmake tasks."
 }
 
 dependencies {
diff --git a/tools/ci_build/github/azure-pipelines/templates/make_java_win_binaries.yml b/tools/ci_build/github/azure-pipelines/templates/make_java_win_binaries.yml
index 9a66615502..0d62ed7907 100644
--- a/tools/ci_build/github/azure-pipelines/templates/make_java_win_binaries.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/make_java_win_binaries.yml
@@ -9,12 +9,16 @@ parameters:
 steps:
     - task: CmdLine@2
       displayName: 'Gradle cmakeCheck'
-      continueOnError:  ${{ parameters.buildOnly }}
       inputs:
-        script: |
-          @echo on
-          call gradlew.bat cmakeCheck -DcmakeBuildDir=$(Build.BinariesDirectory)\RelWithDebInfo --warning-mode all
-        workingDirectory: $(Build.SourcesDirectory)\java
+        ${{ if eq(parameters.buildOnly, true) }}:
+          script: |
+            call gradlew.bat testClasses -DcmakeBuildDir=$(Build.BinariesDirectory)\RelWithDebInfo
+            call gradlew.bat cmakeCheck -x test -DcmakeBuildDir=$(Build.BinariesDirectory)\RelWithDebInfo --warning-mode all
+          workingDirectory: $(Build.SourcesDirectory)\java
+        ${{ else }}:
+          script: |
+            call gradlew.bat cmakeCheck -DcmakeBuildDir=$(Build.BinariesDirectory)\RelWithDebInfo --warning-mode all
+          workingDirectory: $(Build.SourcesDirectory)\java
 
     - task: CmdLine@2
       displayName: 'Add symbols and notices to Java'
diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml
index c726054d8e..52547fd9a7 100644
--- a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml
@@ -196,11 +196,11 @@ stages:
           parameters:
             msbuildPlatform: ${{ parameters.msbuildPlatform }}
             java_artifact_id: ${{ parameters.java_artifact_id }}
-            ${{ if contains(parameters.ort_build_pool_name, 'CPU') }}:
-              buildOnly: false
+            ${{ if or(contains(parameters.buildparameter, 'use_cuda'), contains(parameters.buildparameter, 'use_tensorrt')) }}:
             # When it is a GPU build, we only assemble the java binaries, testing will be done in the later stage with GPU machine
-            ${{ else }}:
               buildOnly: true
+            ${{ else }}:
+              buildOnly: false
 
         - task: PublishBuildArtifacts@1
           displayName: 'Publish Java temp binaries'