[web] utility functions for tensor<->image conversion in ORT web (#13603)

### Description
Data processing capabilities to ORT Web. 
This PR will focus augmenting raw data to and from Tensors.

### Motivation and Context
Enabling different app building use cases to leverage ORT in a more
natural form.
Currently, the user needs to process the data and call Tensor
constructors - these util functions will provide a direct path to
generating ORT tensors.

Co-authored-by: shalvamist <shalva.mist@microsoft.com>

js/common/lib/tensor-impl.ts

@@ -1,7 +1,7 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
-import {Tensor as TensorInterface} from './tensor';
+import {Tensor as TensorInterface, TensorFromImageOptions, TensorToImageDataOptions} from './tensor';
 
 type TensorType = TensorInterface.Type;
 type TensorDataType = TensorInterface.DataType;
@@ -166,6 +166,346 @@ export class Tensor implements TensorInterface {
     this.size = size;
   }
   // #endregion
+  /**
+   * Create a new tensor object from image object
+   *
+   * @param buffer - Extracted image buffer data - assuming RGBA format
+   * @param imageFormat - input image configuration - required configurations height, width, format
+   * @param tensorFormat - output tensor configuration - Default is RGB format
+   */
+  private static bufferToTensor(buffer: Uint8ClampedArray|undefined, options: TensorFromImageOptions): Tensor {
+    if (buffer === undefined) {
+      throw new Error('Image buffer must be defined');
+    }
+    if (options.height === undefined || options.width === undefined) {
+      throw new Error('Image height and width must be defined');
+    }
+
+    const {height, width} = options;
+
+    const norm = options.norm;
+    let normMean: number;
+    let normBias: number;
+    if (norm === undefined || norm.mean === undefined) {
+      normMean = 255;
+    } else {
+      normMean = norm.mean;
+    }
+    if (norm === undefined || norm.bias === undefined) {
+      normBias = 0;
+    } else {
+      normBias = norm.bias;
+    }
+
+    const inputformat = options.bitmapFormat !== undefined ? options.bitmapFormat : 'RGBA';
+    // default value is RGBA since imagedata and HTMLImageElement uses it
+
+    const outputformat = options.tensorFormat !== undefined ?
+        (options.tensorFormat !== undefined ? options.tensorFormat : 'RGB') :
+        'RGB';
+    const offset = height * width;
+    const float32Data = outputformat === 'RGBA' ? new Float32Array(offset * 4) : new Float32Array(offset * 3);
+
+    // Default pointer assignments
+    let step = 4, rImagePointer = 0, gImagePointer = 1, bImagePointer = 2, aImagePointer = 3;
+    let rTensorPointer = 0, gTensorPointer = offset, bTensorPointer = offset * 2, aTensorPointer = -1;
+
+    // Updating the pointer assignments based on the input image format
+    if (inputformat === 'RGB') {
+      step = 3;
+      rImagePointer = 0;
+      gImagePointer = 1;
+      bImagePointer = 2;
+      aImagePointer = -1;
+    }
+
+    // Updating the pointer assignments based on the output tensor format
+    if (outputformat === 'RGBA') {
+      aTensorPointer = offset * 3;
+    } else if (outputformat === 'RBG') {
+      rTensorPointer = 0;
+      bTensorPointer = offset;
+      gTensorPointer = offset * 2;
+    } else if (outputformat === 'BGR') {
+      bTensorPointer = 0;
+      gTensorPointer = offset;
+      rTensorPointer = offset * 2;
+    }
+
+    for (let i = 0; i < offset;
+         i++, rImagePointer += step, bImagePointer += step, gImagePointer += step, aImagePointer += step) {
+      float32Data[rTensorPointer++] = (buffer[rImagePointer] + normBias) / normMean;
+      float32Data[gTensorPointer++] = (buffer[gImagePointer] + normBias) / normMean;
+      float32Data[bTensorPointer++] = (buffer[bImagePointer] + normBias) / normMean;
+      if (aTensorPointer !== -1 && aImagePointer !== -1) {
+        float32Data[aTensorPointer++] = (buffer[aImagePointer] + normBias) / normMean;
+      }
+    }
+
+    // Float32Array -> ort.Tensor
+    const outputTensor = outputformat === 'RGBA' ? new Tensor('float32', float32Data, [1, 4, height, width]) :
+                                                   new Tensor('float32', float32Data, [1, 3, height, width]);
+    return outputTensor;
+  }
+
+  // #region factory
+  static async fromImage(imageData: ImageData, options?: TensorFromImageOptions): Promise<Tensor>;
+  static async fromImage(imageElement: HTMLImageElement, options?: TensorFromImageOptions): Promise<Tensor>;
+  static async fromImage(bitmap: ImageBitmap, options: TensorFromImageOptions): Promise<Tensor>;
+  static async fromImage(url: string, options?: TensorFromImageOptions): Promise<Tensor>;
+
+  static async fromImage(image: ImageData|HTMLImageElement|ImageBitmap|string, options?: TensorFromImageOptions):
+      Promise<Tensor> {
+    // checking the type of image object
+    const isHTMLImageEle = typeof (HTMLImageElement) !== 'undefined' && image instanceof HTMLImageElement;
+    const isImageDataEle = typeof (ImageData) !== 'undefined' && image instanceof ImageData;
+    const isImageBitmap = typeof (ImageBitmap) !== 'undefined' && image instanceof ImageBitmap;
+    const isURL = typeof (String) !== 'undefined' && (image instanceof String || typeof image === 'string');
+
+    let data: Uint8ClampedArray|undefined;
+    let tensorConfig: TensorFromImageOptions = {};
+
+    // filling and checking image configuration options
+    if (isHTMLImageEle) {
+      // HTMLImageElement - image object - format is RGBA by default
+      const canvas = document.createElement('canvas');
+      const pixels2DContext = canvas.getContext('2d');
+
+      if (pixels2DContext != null) {
+        let height = image.naturalHeight;
+        let width = image.naturalWidth;
+
+        if (options !== undefined && options.resizedHeight !== undefined && options.resizedWidth !== undefined) {
+          height = options.resizedHeight;
+          width = options.resizedWidth;
+        }
+
+        if (options !== undefined) {
+          tensorConfig = options;
+          if (options.tensorFormat !== undefined) {
+            throw new Error('Image input config format must be RGBA for HTMLImageElement');
+          } else {
+            tensorConfig.tensorFormat = 'RGBA';
+          }
+          if (options.height !== undefined && options.height !== height) {
+            throw new Error('Image input config height doesn\'t match HTMLImageElement height');
+          } else {
+            tensorConfig.height = height;
+          }
+          if (options.width !== undefined && options.width !== width) {
+            throw new Error('Image input config width doesn\'t match HTMLImageElement width');
+          } else {
+            tensorConfig.width = width;
+          }
+        } else {
+          tensorConfig.tensorFormat = 'RGBA';
+          tensorConfig.height = height;
+          tensorConfig.width = width;
+        }
+
+        canvas.width = width;
+        canvas.height = height;
+
+        pixels2DContext.drawImage(image, 0, 0, width, height);
+        data = pixels2DContext.getImageData(0, 0, width, height).data;
+      } else {
+        throw new Error('Can not access image data');
+      }
+
+    } else if (isImageDataEle) {
+      // ImageData - image object - format is RGBA by default
+      const format = 'RGBA';
+      let height: number;
+      let width: number;
+
+      if (options !== undefined && options.resizedWidth !== undefined && options.resizedHeight !== undefined) {
+        height = options.resizedHeight;
+        width = options.resizedWidth;
+      } else {
+        height = image.height;
+        width = image.width;
+      }
+
+      if (options !== undefined) {
+        tensorConfig = options;
+        if (options.bitmapFormat !== undefined && options.bitmapFormat !== format) {
+          throw new Error('Image input config format must be RGBA for ImageData');
+        } else {
+          tensorConfig.bitmapFormat = 'RGBA';
+        }
+      } else {
+        tensorConfig.bitmapFormat = 'RGBA';
+      }
+
+      tensorConfig.height = height;
+      tensorConfig.width = width;
+
+      if (options !== undefined) {
+        const tempCanvas = document.createElement('canvas');
+
+        tempCanvas.width = width;
+        tempCanvas.height = height;
+
+        const pixels2DContext = tempCanvas.getContext('2d');
+
+        if (pixels2DContext != null) {
+          pixels2DContext.putImageData(image, 0, 0);
+          data = pixels2DContext.getImageData(0, 0, width, height).data;
+        } else {
+          throw new Error('Can not access image data');
+        }
+      } else {
+        data = image.data;
+      }
+
+    } else if (isImageBitmap) {
+      // ImageBitmap - image object - format must be provided by user
+      if (options === undefined) {
+        throw new Error('Please provide image config with format for Imagebitmap');
+      }
+      if (options.bitmapFormat !== undefined) {
+        throw new Error('Image input config format must be defined for ImageBitmap');
+      }
+
+      const pixels2DContext = document.createElement('canvas').getContext('2d');
+
+      if (pixels2DContext != null) {
+        const height = image.height;
+        const width = image.width;
+        pixels2DContext.drawImage(image, 0, 0, width, height);
+        data = pixels2DContext.getImageData(0, 0, width, height).data;
+        if (options !== undefined) {
+          // using square brackets to avoid TS error - type 'never'
+          if (options.height !== undefined && options.height !== height) {
+            throw new Error('Image input config height doesn\'t match ImageBitmap height');
+          } else {
+            tensorConfig.height = height;
+          }
+          // using square brackets to avoid TS error - type 'never'
+          if (options.width !== undefined && options.width !== width) {
+            throw new Error('Image input config width doesn\'t match ImageBitmap width');
+          } else {
+            tensorConfig.width = width;
+          }
+        } else {
+          tensorConfig.height = height;
+          tensorConfig.width = width;
+        }
+        return Tensor.bufferToTensor(data, tensorConfig);
+      } else {
+        throw new Error('Can not access image data');
+      }
+
+    } else if (isURL) {
+      return new Promise((resolve, reject) => {
+        const canvas = document.createElement('canvas');
+        const context = canvas.getContext('2d');
+        if (!image || !context) {
+          return reject();
+        }
+        const newImage = new Image();
+        newImage.crossOrigin = 'Anonymous';
+        newImage.src = image as string;
+        newImage.onload = () => {
+          canvas.width = newImage.width;
+          canvas.height = newImage.height;
+          context.drawImage(newImage, 0, 0, canvas.width, canvas.height);
+          const img = context.getImageData(0, 0, canvas.width, canvas.height);
+          if (options !== undefined) {
+            // using square brackets to avoid TS error - type 'never'
+            if (options.height !== undefined && options.height !== canvas.height) {
+              throw new Error('Image input config height doesn\'t match ImageBitmap height');
+            } else {
+              tensorConfig.height = canvas.height;
+            }
+            // using square brackets to avoid TS error - type 'never'
+            if (options.width !== undefined && options.width !== canvas.width) {
+              throw new Error('Image input config width doesn\'t match ImageBitmap width');
+            } else {
+              tensorConfig.width = canvas.width;
+            }
+          } else {
+            tensorConfig.height = canvas.height;
+            tensorConfig.width = canvas.width;
+          }
+          resolve(Tensor.bufferToTensor(img.data, tensorConfig));
+        };
+      });
+    } else {
+      throw new Error('Input data provided is not supported - aborted tensor creation');
+    }
+
+    if (data !== undefined) {
+      return Tensor.bufferToTensor(data, tensorConfig);
+    } else {
+      throw new Error('Input data provided is not supported - aborted tensor creation');
+    }
+  }
+
+  toImageData(options?: TensorToImageDataOptions): ImageData {
+    const pixels2DContext = document.createElement('canvas').getContext('2d');
+    let image: ImageData;
+    if (pixels2DContext != null) {
+      // Default values for height and width & format
+      const width = this.dims[3];
+      const height = this.dims[2];
+      const channels = this.dims[1];
+
+      const inputformat = options !== undefined ? (options.format !== undefined ? options.format : 'RGB') : 'RGB';
+      const normMean = options !== undefined ? (options.norm?.mean !== undefined ? options.norm.mean : 255) : 255;
+      const normBias = options !== undefined ? (options.norm?.bias !== undefined ? options.norm.bias : 0) : 0;
+      const offset = height * width;
+
+      if (options !== undefined) {
+        if (options.height !== undefined && options.height !== height) {
+          throw new Error('Image output config height doesn\'t match tensor height');
+        }
+        if (options.width !== undefined && options.width !== width) {
+          throw new Error('Image output config width doesn\'t match tensor width');
+        }
+        if (options.format !== undefined && (channels === 4 && options.format !== 'RGBA') ||
+            (channels === 3 && (options.format !== 'RGB' && options.format !== 'BGR'))) {
+          throw new Error('Tensor format doesn\'t match input tensor dims');
+        }
+      }
+
+      // Default pointer assignments
+      const step = 4;
+      let rImagePointer = 0, gImagePointer = 1, bImagePointer = 2, aImagePointer = 3;
+      let rTensorPointer = 0, gTensorPointer = offset, bTensorPointer = offset * 2, aTensorPointer = -1;
+
+      // Updating the pointer assignments based on the input image format
+      if (inputformat === 'RGBA') {
+        rTensorPointer = 0;
+        gTensorPointer = offset;
+        bTensorPointer = offset * 2;
+        aTensorPointer = offset * 3;
+      } else if (inputformat === 'RGB') {
+        rTensorPointer = 0;
+        gTensorPointer = offset;
+        bTensorPointer = offset * 2;
+      } else if (inputformat === 'RBG') {
+        rTensorPointer = 0;
+        bTensorPointer = offset;
+        gTensorPointer = offset * 2;
+      }
+
+      image = pixels2DContext.createImageData(width, height);
+
+      for (let i = 0; i < height * width;
+           rImagePointer += step, gImagePointer += step, bImagePointer += step, aImagePointer += step, i++) {
+        image.data[rImagePointer] = ((this.data[rTensorPointer++] as number) - normBias) * normMean;  // R value
+        image.data[gImagePointer] = ((this.data[gTensorPointer++] as number) - normBias) * normMean;  // G value
+        image.data[bImagePointer] = ((this.data[bTensorPointer++] as number) - normBias) * normMean;  // B value
+        image.data[aImagePointer] =
+            aTensorPointer === -1 ? 255 : ((this.data[aTensorPointer++] as number) - normBias) * normMean;  // A value
+      }
+
+    } else {
+      throw new Error('Can not access image data');
+    }
+    return image;
+  }
 
   // #region fields
   readonly dims: readonly number[];

js/common/lib/tensor-utils.ts

@@ -1,7 +1,7 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
-import {Tensor, TypedTensor} from './tensor';
+import {Tensor, TensorToImageDataOptions, TypedTensor} from './tensor';
 
 interface Properties {
   /**
@@ -20,4 +20,13 @@ export interface TypedShapeUtils<T extends Tensor.Type> {
 }
 
 // TODO: add more tensor utilities
-export interface TypedTensorUtils<T extends Tensor.Type> extends Properties, TypedShapeUtils<T> {}
+export interface TypedTensorUtils<T extends Tensor.Type> extends Properties, TypedShapeUtils<T> {
+  /**
+   * creates an ImageData instance from tensor
+   *
+   * @param tensorFormat - Interface describing tensor instance - Defaults: RGB, 3 channels, 0-255, NHWC
+   * 0-255, NHWC
+   * @returns An ImageData instance which can be used to draw on canvas
+   */
+  toImageData(options?: TensorToImageDataOptions): ImageData;
+}

js/common/lib/tensor.ts

@@ -234,5 +234,124 @@ export interface TensorConstructor {
   // #endregion
 }
 
+/**
+ * Specify the image format. Assume 'RGBA' if omitted.
+ */
+export type ImageFormat = 'RGB'|'RGBA'|'BGR'|'RBG';
+
+/**
+ * Describes Tensor configuration to an image data.
+ */
+export interface TensorToImageDataOptions {
+  /**
+   * Describes Tensor channels order.
+   */
+  format?: ImageFormat;
+  /**
+   * Tensor channel layout - default is 'NHWC'
+   */
+  tensorLayout?: 'NHWC'|'NCHW';
+  /**
+   * Describes Tensor Height - can be accessed via tensor dimensions as well
+   */
+  height?: number;
+  /**
+   * Describes Tensor Width - can be accessed via tensor dimensions as well
+   */
+  width?: number;
+  /**
+   * Describes normalization parameters to ImageData conversion from tensor - default values - Bias: 0, Mean: 255
+   */
+  norm?: {
+    bias?: number;  // Todo add support - |[number,number,number]|[number,number,number,number];
+    mean?: number;  // Todo add support - |[number,number,number]|[number,number,number,number];
+  };
+}
+/**
+ * Describes Tensor and Image configuration to an image data.
+ */
+export interface TensorFromImageOptions {
+  /**
+   * Describes image data format - will be used only in the case of ImageBitMap
+   */
+  bitmapFormat?: ImageFormat;
+  /**
+   * Describes Tensor channels order - can differ from original image
+   */
+  tensorFormat?: ImageFormat;
+  /**
+   * Tensor data type - default is 'float32'
+   */
+  dataType?: 'float32'|'uint8';
+  /**
+   * Tensor channel layout - default is 'NHWC'
+   */
+  tensorLayout?: 'NHWC'|'NCHW';
+  /**
+   * Describes Image Height - Required only in the case of ImageBitMap
+   */
+  height?: number;
+  /**
+   * Describes Image Width - Required only in the case of ImageBitMap
+   */
+  width?: number;
+  /**
+   * Describes resized height - can be accessed via tensor dimensions as well
+   */
+  resizedHeight?: number;
+  /**
+   * Describes resized width - can be accessed via tensor dimensions as well
+   */
+  resizedWidth?: number;
+  /**
+   * Describes normalization parameters to tensor conversion from image data - default values - Bias: 0, Mean: 255
+   */
+  norm?: {
+    bias?: number;  // Todo add support - |[number,number,number]|[number,number,number,number];
+    mean?: number;  // Todo add support - |[number,number,number]|[number,number,number,number];
+  };
+}
+export interface TensorFactory {
+  /**
+   * create a tensor from image object - HTMLImageElement, ImageData, ImageBitmap, URL
+   *
+   * @param imageData - {ImageData} - composed of: Uint8ClampedArray, width. height - uses known pixel format RGBA
+   * @param options - Optional - Interface describing input image & output tensor -
+   * Input Defaults: RGBA, 3 channels, 0-255, NHWC - Output Defaults: same as input parameters
+   * @returns A promise that resolves to a tensor object
+   */
+  fromImage(imageData: ImageData, options?: TensorFromImageOptions): Promise<Tensor>;
+
+  /**
+   * create a tensor from image object - HTMLImageElement, ImageData, ImageBitmap, URL
+   *
+   * @param imageElement - {HTMLImageElement} - since the data is stored as ImageData no need for format parameter
+   * @param options - Optional - Interface describing input image & output tensor -
+   * Input Defaults: RGBA, 3 channels, 0-255, NHWC - Output Defaults: same as input parameters
+   * @returns A promise that resolves to a tensor object
+   */
+  fromImage(imageElement: HTMLImageElement, options?: TensorFromImageOptions): Promise<Tensor>;
+
+  /**
+   * create a tensor from image object - HTMLImageElement, ImageData, ImageBitmap, URL
+   *
+   * @param url - {string} - Assuming the string is a URL to an image
+   * @param options - Optional - Interface describing input image & output tensor -
+   * Input Defaults: RGBA, 3 channels, 0-255, NHWC - Output Defaults: same as input parameters
+   * @returns A promise that resolves to a tensor object
+   */
+  fromImage(url: string, options?: TensorFromImageOptions): Promise<Tensor>;
+
+  /**
+   * create a tensor from image object - HTMLImageElement, ImageData, ImageBitmap, URL
+   *
+   * @param bitMap - {ImageBitmap} - since the data is stored as ImageData no need for format parameter
+   * @param options - NOT Optional - Interface describing input image & output tensor -
+   * Output Defaults: same as input parameters
+   * @returns A promise that resolves to a tensor object
+   */
+  fromImage(bitmap: ImageBitmap, options: TensorFromImageOptions): Promise<Tensor>;
+}
+
 // eslint-disable-next-line @typescript-eslint/naming-convention
 export const Tensor = TensorImpl as TensorConstructor;

js/common/tsconfig.json

@@ -3,7 +3,7 @@
   "compilerOptions": {
     "outDir": "dist/lib",
     "esModuleInterop": false,
-    "noUnusedParameters": true
+    "noUnusedParameters": true,
   },
   "include": ["lib"]
 }

js/package.json

@@ -1,6 +1,7 @@
 {
   "devDependencies": {
     "@types/fs-extra": "^9.0.13",
+    "@types/node": "^18.11.18",
     "@types/npmlog": "^4.1.4",
     "@typescript-eslint/eslint-plugin": "^4.22.1",
     "@typescript-eslint/parser": "^4.22.1",

js/tsconfig.json

@@ -5,7 +5,7 @@
     "declaration": true,
     "esModuleInterop": true,
     "target": "ES2017",
-    "lib": ["ES2017", "ESNext.BigInt"],
+    "lib": ["ES2017", "ESNext.BigInt", "dom"],
     "sourceMap": true,
     "noUnusedLocals": true,
     "noImplicitAny": true,

js/web/lib/wasm/proxy-wrapper.ts

@@ -115,7 +115,7 @@ export const initWasm = async(): Promise<void> => {
     // overwrite wasm filepaths
     if (env.wasm.wasmPaths === undefined) {
       if (scriptSrc && scriptSrc.indexOf('blob:') !== 0) {
-        env.wasm.wasmPaths = scriptSrc.substr(0, (scriptSrc as string).lastIndexOf('/') + 1);
+        env.wasm.wasmPaths = scriptSrc.substr(0, +(scriptSrc).lastIndexOf('/') + 1);
       }
     }
 

js/web/tsconfig.json

@@ -4,7 +4,6 @@
     "module": "CommonJS",
     "downlevelIteration": true,
     "declarationDir": "./types",
-    "lib": ["DOM"]
   },
   "include": ["lib", "script", "test"],
   "exclude": ["lib/wasm/proxy-worker"]