From e4a985ff17a0be0b7b39050a9890f5fa1f28ee45 Mon Sep 17 00:00:00 2001 From: Du Li Date: Tue, 18 May 2021 06:31:00 -0700 Subject: [PATCH] [JS/Web] WebGL Profiling Tool (#7724) --- .../backends/webgl/inference-handler.ts | 3 ++ js/web/lib/onnxjs/backends/webgl/ops/conv.ts | 2 + .../onnxjs/backends/webgl/ops/im2col-pack.ts | 1 + .../webgl/ops/instance-normalization.ts | 6 ++- .../onnxjs/backends/webgl/ops/matmul-pack.ts | 1 + js/web/lib/onnxjs/backends/webgl/ops/pack.ts | 1 + .../backends/webgl/ops/reshape-packed.ts | 1 + .../lib/onnxjs/backends/webgl/ops/softmax.ts | 3 ++ js/web/lib/onnxjs/backends/webgl/ops/split.ts | 1 + .../onnxjs/backends/webgl/ops/uint8-encode.ts | 3 +- .../lib/onnxjs/backends/webgl/ops/unpack.ts | 1 + .../onnxjs/backends/webgl/program-manager.ts | 7 +--- js/web/lib/onnxjs/backends/webgl/types.ts | 1 + .../onnxjs/backends/webgl/webgl-context.ts | 1 + js/web/lib/onnxjs/execution-plan.ts | 11 +---- js/web/script/parse-profiler.ts | 42 +++++++++++++++++++ 16 files changed, 67 insertions(+), 18 deletions(-) create mode 100644 js/web/script/parse-profiler.ts diff --git a/js/web/lib/onnxjs/backends/webgl/inference-handler.ts b/js/web/lib/onnxjs/backends/webgl/inference-handler.ts index 7e09fbe4e7..778da82276 100644 --- a/js/web/lib/onnxjs/backends/webgl/inference-handler.ts +++ b/js/web/lib/onnxjs/backends/webgl/inference-handler.ts @@ -32,6 +32,9 @@ export class WebGLInferenceHandler implements InferenceHandler { let artifact = this.session.programManager.getArtifact(op); if (!artifact) { const programInfo = op.createProgramInfo(this, inputs); + if (!programInfo.name) { + programInfo.name = op.constructor?.name; + } artifact = this.session.programManager.build(programInfo); this.session.programManager.setArtifact(op, artifact); } diff --git a/js/web/lib/onnxjs/backends/webgl/ops/conv.ts b/js/web/lib/onnxjs/backends/webgl/ops/conv.ts index 9012b07a68..8fdb052ceb 100644 --- a/js/web/lib/onnxjs/backends/webgl/ops/conv.ts +++ b/js/web/lib/onnxjs/backends/webgl/ops/conv.ts @@ -299,6 +299,7 @@ export class WebGLUnpackedConv extends Conv { } `; return { + name: 'Im2Col', inputLayouts: [inferenceHandler.createTextureLayoutFromShape(xshape)], outputLayout, samplers: ['X'], @@ -361,6 +362,7 @@ export class WebGLUnpackedConv extends Conv { return value; }`; return { + name: 'dotProduct', inputLayouts: inputs.length === 3 ? [im2colLayout, kLayout, bLayout!] : [im2colLayout, kLayout], outputLayout, shaderSource, diff --git a/js/web/lib/onnxjs/backends/webgl/ops/im2col-pack.ts b/js/web/lib/onnxjs/backends/webgl/ops/im2col-pack.ts index ea172cecfa..4fa44e7189 100644 --- a/js/web/lib/onnxjs/backends/webgl/ops/im2col-pack.ts +++ b/js/web/lib/onnxjs/backends/webgl/ops/im2col-pack.ts @@ -82,6 +82,7 @@ export class WebGLIm2ColPacked implements WebGLOperator { } `; return { + name: 'WebGLIm2ColPacked', inputLayouts: [inferenceHandler.getOrCreateTextureLayout(inputs[0], 4, true, xshape, true)], outputLayout: inferenceHandler.createTextureLayoutFromShape(im2colShape, 4, im2colShape, {isPacked: true, reverseWH: true}), diff --git a/js/web/lib/onnxjs/backends/webgl/ops/instance-normalization.ts b/js/web/lib/onnxjs/backends/webgl/ops/instance-normalization.ts index fbe00c6fed..b6670e344e 100644 --- a/js/web/lib/onnxjs/backends/webgl/ops/instance-normalization.ts +++ b/js/web/lib/onnxjs/backends/webgl/ops/instance-normalization.ts @@ -12,8 +12,8 @@ export class WebGLInstanceNormalization extends InstanceNormalization { if (!this.artifacts) { this.artifacts = []; const programInfos = this.createProgramInfos(inferenceHandler, inputs); - programInfos.forEach((pi) => { - const artifact = inferenceHandler.session.programManager.build(pi); + programInfos.forEach((programInfo) => { + const artifact = inferenceHandler.session.programManager.build(programInfo); this.artifacts.push(artifact); }); } @@ -78,6 +78,7 @@ export class WebGLInstanceNormalization extends InstanceNormalization { outputLayout: inferenceHandler.createTextureLayoutFromShape(outputShape, 4, outputUnpackedShape), samplers: ['X'], shaderSource, + name: 'MeanAndVariance', }; } @@ -114,6 +115,7 @@ export class WebGLInstanceNormalization extends InstanceNormalization { samplers: ['X', 'MeanAndVariance', 'Scale', 'B'], variables: [{name: 'epsilon', type: 'float'}], shaderSource, + name: 'ComputOutput', }; } createProgramInfos(inferenceHandler: WebGLInferenceHandler, inputs: Tensor[]): ProgramInfo[] { diff --git a/js/web/lib/onnxjs/backends/webgl/ops/matmul-pack.ts b/js/web/lib/onnxjs/backends/webgl/ops/matmul-pack.ts index a77d065378..70535184db 100644 --- a/js/web/lib/onnxjs/backends/webgl/ops/matmul-pack.ts +++ b/js/web/lib/onnxjs/backends/webgl/ops/matmul-pack.ts @@ -49,6 +49,7 @@ export class WebGLMatMulPacked extends MatMul implements WebGLOperator { return value; }`; return { + name: 'WebGLMatMulPacked', inputLayouts: inputs.map((t, i) => handler.getOrCreateTextureLayout(t, 4, true, inputs[i].dims, true)), outputLayout: handler.createTextureLayoutFromShape(outputShape, 4, outputShape, {isPacked: true, reverseWH: true}), diff --git a/js/web/lib/onnxjs/backends/webgl/ops/pack.ts b/js/web/lib/onnxjs/backends/webgl/ops/pack.ts index 5928daba98..2f3a9778e5 100644 --- a/js/web/lib/onnxjs/backends/webgl/ops/pack.ts +++ b/js/web/lib/onnxjs/backends/webgl/ops/pack.ts @@ -57,6 +57,7 @@ export class WebGLPack implements WebGLOperator { `; return { + name: 'WebGLPack', inputLayouts: [handler.getOrCreateTextureLayout(inputs[0], 1, false, [], true)], outputLayout, samplers: ['A'], diff --git a/js/web/lib/onnxjs/backends/webgl/ops/reshape-packed.ts b/js/web/lib/onnxjs/backends/webgl/ops/reshape-packed.ts index 49f09819a1..8ef3474231 100644 --- a/js/web/lib/onnxjs/backends/webgl/ops/reshape-packed.ts +++ b/js/web/lib/onnxjs/backends/webgl/ops/reshape-packed.ts @@ -106,6 +106,7 @@ export class WebGLReshapePacked extends Reshape implements WebGLOperator { `; return { + name: 'WebGLReshapePacked', inputLayouts: [inputLayout], outputLayout: this.outputLayout, samplers: ['A'], diff --git a/js/web/lib/onnxjs/backends/webgl/ops/softmax.ts b/js/web/lib/onnxjs/backends/webgl/ops/softmax.ts index 763ec774d8..25f99583ca 100644 --- a/js/web/lib/onnxjs/backends/webgl/ops/softmax.ts +++ b/js/web/lib/onnxjs/backends/webgl/ops/softmax.ts @@ -75,6 +75,7 @@ export class WebGLSoftmax extends Softmax { outputLayout: inferenceHandler.createTextureLayoutFromShape(outputShape), samplers: ['A', 'Max', 'Norm'], shaderSource, + name: 'SoftMax', }; } @@ -131,6 +132,7 @@ export class WebGLSoftmax extends Softmax { outputLayout: inferenceHandler.createTextureLayoutFromShape(outputShape), samplers: ['A', 'Max'], shaderSource, + name: 'ComputScale', }; } /** @@ -179,6 +181,7 @@ export class WebGLSoftmax extends Softmax { outputLayout: inferenceHandler.createTextureLayoutFromShape(outputShape), samplers: ['A'], shaderSource, + name: 'ComputeMax', }; } createProgramInfos(inferenceHandler: WebGLInferenceHandler, inputs: Tensor[]): ProgramInfo[] { diff --git a/js/web/lib/onnxjs/backends/webgl/ops/split.ts b/js/web/lib/onnxjs/backends/webgl/ops/split.ts index 41a1eab020..c453d92ca4 100644 --- a/js/web/lib/onnxjs/backends/webgl/ops/split.ts +++ b/js/web/lib/onnxjs/backends/webgl/ops/split.ts @@ -43,6 +43,7 @@ export class WebGLSplit extends Split { return _A(indices); }`; return { + name: 'WebGLSplit', inputLayouts: [inferenceHandler.getOrCreateTextureLayout(input)], outputLayout: inferenceHandler.createTextureLayoutFromShape(outputShape), samplers: ['A'], diff --git a/js/web/lib/onnxjs/backends/webgl/ops/uint8-encode.ts b/js/web/lib/onnxjs/backends/webgl/ops/uint8-encode.ts index a144fc691f..0baea4eaca 100644 --- a/js/web/lib/onnxjs/backends/webgl/ops/uint8-encode.ts +++ b/js/web/lib/onnxjs/backends/webgl/ops/uint8-encode.ts @@ -71,7 +71,8 @@ export class WebGLUint8Encode { float value = ${glsl.texture2D}(X,TexCoords).r; ${glsl.output} = encodeAsUint8(value); }`; - const programInfo = {inputLayouts: [input], outputLayout, samplers: ['X'], shaderSource, hasMain: true}; + const programInfo = + {name: 'Uint8Encode', inputLayouts: [input], outputLayout, samplers: ['X'], shaderSource, hasMain: true}; const artifact = inferenceHandler.session.programManager.build(programInfo); const encoder = inferenceHandler.session.backend.glContext.getEncoder('byte', 4); diff --git a/js/web/lib/onnxjs/backends/webgl/ops/unpack.ts b/js/web/lib/onnxjs/backends/webgl/ops/unpack.ts index e62d3083ed..f88fff2659 100644 --- a/js/web/lib/onnxjs/backends/webgl/ops/unpack.ts +++ b/js/web/lib/onnxjs/backends/webgl/ops/unpack.ts @@ -48,6 +48,7 @@ export class WebGLUnpack implements WebGLOperator { `; return { + name: 'WebGLUnpack', inputLayouts: [handler.getOrCreateTextureLayout(inputs[0], 4, true, inputs[0].dims, true)], outputLayout, samplers: ['A'], diff --git a/js/web/lib/onnxjs/backends/webgl/program-manager.ts b/js/web/lib/onnxjs/backends/webgl/program-manager.ts index 4fde126d71..aa98d7ce02 100644 --- a/js/web/lib/onnxjs/backends/webgl/program-manager.ts +++ b/js/web/lib/onnxjs/backends/webgl/program-manager.ts @@ -37,10 +37,7 @@ export class ProgramManager { this.repo.set(key, artifact); } run(buildArtifact: Artifact, runData: RunData): void { - const inputInfo = runData.inputTextureDatas.map((d, i) => `input${i}:[${d.shape}]`).join(', '); - const outputInfo = `output: [${runData.outputTextureData.shape}]`; - - this.profiler.event('backend', `ProgramManager.run ${inputInfo} ; ${outputInfo}`, () => { + this.profiler.event('op', `ProgramManager.run ${buildArtifact.programInfo.name ?? 'unknown kernel'}`, () => { const gl = this.glContext.gl; const program = buildArtifact.program; gl.useProgram(program); @@ -57,7 +54,7 @@ export class ProgramManager { this.profiler.event('backend', 'GlContext.draw()', () => { this.doDraw(buildArtifact, runData); }); - }); + }, this.glContext); } dispose(): void { if (this.vertexShader) { diff --git a/js/web/lib/onnxjs/backends/webgl/types.ts b/js/web/lib/onnxjs/backends/webgl/types.ts index 0d0cf4feaf..40d98e7689 100644 --- a/js/web/lib/onnxjs/backends/webgl/types.ts +++ b/js/web/lib/onnxjs/backends/webgl/types.ts @@ -82,6 +82,7 @@ export interface ProgramInfo { expectPackedInputs?: boolean; expectPackedOutputs?: boolean; + name?: string; } export interface VariableInfo { diff --git a/js/web/lib/onnxjs/backends/webgl/webgl-context.ts b/js/web/lib/onnxjs/backends/webgl/webgl-context.ts index 26f5f49605..c859fddc5f 100644 --- a/js/web/lib/onnxjs/backends/webgl/webgl-context.ts +++ b/js/web/lib/onnxjs/backends/webgl/webgl-context.ts @@ -520,6 +520,7 @@ export class WebGLContext { if (this.version === 2) { const gl2 = this.gl as WebGL2RenderingContext; timeElapsed = gl2.getQueryParameter(query, gl2.QUERY_RESULT); + gl2.deleteQuery(query); } else { // TODO: add webgl 1 handling. throw new Error('WebGL1 profiling currently not supported'); diff --git a/js/web/lib/onnxjs/execution-plan.ts b/js/web/lib/onnxjs/execution-plan.ts index 3d6666e6c8..fe55667443 100644 --- a/js/web/lib/onnxjs/execution-plan.ts +++ b/js/web/lib/onnxjs/execution-plan.ts @@ -2,8 +2,6 @@ // Licensed under the MIT License. import {SessionHandler} from './backend'; -import {WebGLBackend} from './backends/backend-webgl'; -import {WebGLContext} from './backends/webgl/webgl-context'; import {Graph} from './graph'; import {Logger, Profiler} from './instrument'; import {Operator} from './operators'; @@ -53,12 +51,6 @@ export class ExecutionPlan { } async execute(sessionHandler: SessionHandler, modelInputs: Tensor[]): Promise { - const isWebGLBackend = sessionHandler.backend instanceof WebGLBackend; - let glCtx: WebGLContext|undefined; - if (isWebGLBackend) { - glCtx = (sessionHandler.backend as WebGLBackend).glContext; - } - return this.profiler.event('session', 'ExecutionPlan.execute', async () => { // reset mediem result this.reset(); @@ -114,8 +106,7 @@ export class ExecutionPlan { return result; }; - const outputList = isWebGLBackend ? await this.profiler.event('node', thisOp.node.name, execNodeFn, glCtx) : - await this.profiler.event('node', thisOp.node.name, execNodeFn); + const outputList = await this.profiler.event('node', thisOp.node.name, execNodeFn); // check output if (outputList.length !== thisOp.node.outputs.length) { diff --git a/js/web/script/parse-profiler.ts b/js/web/script/parse-profiler.ts new file mode 100644 index 0000000000..674be5cf8e --- /dev/null +++ b/js/web/script/parse-profiler.ts @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + + +/* eslint-disable @typescript-eslint/restrict-plus-operands */ + +// parse-profiler +// +// this script is used to parse performance profiling result. +// usage: +// STEP.1 - profiling +// > npm test -- model test/test-data/{path-to-my-model} --backend={cpu/webgl/wasm} --profile > profile.raw.log +// STEP.2 - parse +// > node script/parse-profiler < profile.raw.log > profile.parsed.log + + +import * as readline from 'readline'; +const lines = readline.createInterface({input: process.stdin, output: process.stdout, terminal: false}); + +// eslint-disable-next-line no-control-regex +const matcher = /Profiler\.([^[\s\x1b]+)(\x1b\[0m)? (\d.+Z)\|([\d.]+)ms on event '([^']+)' at (\d*\.*\d*)/; + +const allEvents: any[] = []; +lines.on('line', input => { + const matches = matcher.exec(input); + if (matches) { + // console.log(matches); + const category = matches[1]; + const logTimeStamp = new Date(matches[3]); + const ms = Number.parseFloat(matches[4]); + const event = matches[5]; + const endTimeInNumber = matches[6]; + allEvents.push({event, ms, logTimeStamp, category, endTimeInNumber}); + } +}); + +lines.on('close', () => { + for (const i of allEvents) { + console.log(`${(i.category + ' ').substring(0, 12)} ${((i.ms) + ' ').substring(0, 12)} ${ + (i.event + ' ').substring(0, 40)} ${i.endTimeInNumber}`); + } +});