From 307ad1413a0c304beff6c328f1fe6c2e7317ddeb Mon Sep 17 00:00:00 2001
From: Seungwon Jeong <jsw1295@gmail.com>
Date: Tue, 22 Nov 2022 05:03:48 +0900
Subject: [PATCH] [js/web] support 'pytorch_half_pixel' mode for WebGL kernel
 'Resize' (#11208)

**Description**:
1. add pytorch_half_pixel interpolation mode in resize-packed.ts
Changes: add the following case in createPackedResizeProgramInfo
function:
```
case 'pytorch_half_pixel':
          getSourceFracIndex = `
                    vec4 getSourceFracIndex(ivec4 coords) {
                        vec4 fcoords = vec4(coords);
                        return vec4(
                            ${outputWidth}.0 > 1.0 ? (fcoords.x + 0.5) / scaleWHWH.x - 0.5 : 0.0,
                            ${outputHeight}.0 > 1.0 ? (fcoords.y + 0.5) / scaleWHWH.y - 0.5 : 0.0,
                            ${outputWidth}.0 > 1.0 ? (fcoords.z + 0.5) / scaleWHWH.z - 0.5 : 0.0,
                            ${outputHeight}.0 > 1.0 ? (fcoords.w + 0.5) / scaleWHWH.w - 0.5 : 0.0
                          );
                    }
                `;
          break;
```
2. fix "unrecognized input '' for node: Resize_$num" error when inputs
like [input_tensor, None, scale_factor] (roiInput not given) are fed
into the resize layer.
Changes: change in input handling logic in upsample.ts & node scanning
logic in graph.ts

**Motivation and Context**
Before this fix, we aren't able to use webGL backend when the neural
network contains pytorch resize layers. This fix adds
'pytorch_half_pixel' interpolation mode support and makes it possible to
use webGL backend for more kind of computer vision networks.

This commit solves:
#10430

Co-authored-by: neo <neo@icode-lab.com>
Co-authored-by: Yulong Wang <7679871+fs-eire@users.noreply.github.com>
---
 .../lib/onnxjs/backends/webgl/ops/resize-packed.ts | 13 +++++++++++++
 js/web/lib/onnxjs/backends/webgl/ops/upsample.ts   | 14 ++++++++++----
 js/web/lib/onnxjs/graph.ts                         |  4 ++++
 3 files changed, 27 insertions(+), 4 deletions(-)

diff --git a/js/web/lib/onnxjs/backends/webgl/ops/resize-packed.ts b/js/web/lib/onnxjs/backends/webgl/ops/resize-packed.ts
index 028ef3b2a1..c0d485d95f 100644
--- a/js/web/lib/onnxjs/backends/webgl/ops/resize-packed.ts
+++ b/js/web/lib/onnxjs/backends/webgl/ops/resize-packed.ts
@@ -95,6 +95,19 @@ const createPackedResizeProgramInfo =
                     }
                 `;
           break;
+        case 'pytorch_half_pixel':
+          getSourceFracIndex = `
+                    vec4 getSourceFracIndex(ivec4 coords) {
+                        vec4 fcoords = vec4(coords);
+                        return vec4(
+                            ${outputWidth}.0 > 1.0 ? (fcoords.x + 0.5) / scaleWHWH.x - 0.5 : 0.0,
+                            ${outputHeight}.0 > 1.0 ? (fcoords.y + 0.5) / scaleWHWH.y - 0.5 : 0.0,
+                            ${outputWidth}.0 > 1.0 ? (fcoords.z + 0.5) / scaleWHWH.z - 0.5 : 0.0,
+                            ${outputHeight}.0 > 1.0 ? (fcoords.w + 0.5) / scaleWHWH.w - 0.5 : 0.0
+                          );
+                    }
+                `;
+          break;
         case 'align_corners':
           getSourceFracIndex = `
                     vec4 getSourceFracIndex(ivec4 coords) {
diff --git a/js/web/lib/onnxjs/backends/webgl/ops/upsample.ts b/js/web/lib/onnxjs/backends/webgl/ops/upsample.ts
index 7ec20edb85..d7bb1393d2 100644
--- a/js/web/lib/onnxjs/backends/webgl/ops/upsample.ts
+++ b/js/web/lib/onnxjs/backends/webgl/ops/upsample.ts
@@ -99,9 +99,15 @@ export const parseUpsampleAttributes = (node: Graph.Node, opset: number): Upsamp
   let sizesInputIdx = 0;
 
   if (opset > 10) {
-    roiInputIdx = 1;
-    scalesInputIdx = 2;
-    sizesInputIdx = 3;
+    // handle when roiInput is not given
+    if (node.inputs.length > 2) {
+      roiInputIdx = 1;
+      scalesInputIdx = 2;
+      sizesInputIdx = 3;
+    } else {
+      scalesInputIdx = 1;
+      sizesInputIdx = 2;
+    }
   } else if (opset === 9) {
     scalesInputIdx = 1;
   }
@@ -307,7 +313,7 @@ const createUpsampleProgramInfo =
 export const validateInputs = (inputs: Tensor[], attribute: UpsampleAttributes): void => {
   if (!inputs || (attribute.opset < 9 && inputs.length !== 1) ||
       (attribute.opset >= 9 && attribute.opset < 11 && inputs.length !== 2) ||
-      (attribute.opset >= 11 && inputs.length !== 3 && inputs.length !== 4)) {
+      (attribute.opset >= 11 && inputs.length < 2)) {
     throw new Error('invalid inputs.');
   }
 
diff --git a/js/web/lib/onnxjs/graph.ts b/js/web/lib/onnxjs/graph.ts
index 6816b7acd9..779b4e60cd 100644
--- a/js/web/lib/onnxjs/graph.ts
+++ b/js/web/lib/onnxjs/graph.ts
@@ -333,6 +333,10 @@ class GraphImpl implements Graph, Graph.Transformer {
       for (const input of nodeProto.input) {
         const dataIndex = dataIndices.get(input);
         if (typeof dataIndex === 'undefined') {
+          // handle exception when opset > 9 and roi not given
+          if (input === '' && nodeProto.input.length === 3 && nodeProto.opType === 'Resize') {
+            continue;
+          }
           throw new Error(`unrecognized input '${input}' for node: ${nodeProto.name}`);
         }
         node.inputs.push(dataIndex);