diff --git a/src/routes/blogs/+page.svelte b/src/routes/blogs/+page.svelte index 73e0adfb5a..b128ed789e 100644 --- a/src/routes/blogs/+page.svelte +++ b/src/routes/blogs/+page.svelte @@ -10,6 +10,7 @@ import LlamaImage from '../../images/blogs/accelerating-llama-2/Figure1-LLaMA-2-7B-E2E-Throughput.png'; import SDXLTurboImage from '../../images/blogs/sdxl_blog_thumbnail.png'; import Phi2Image from '../../routes/blogs/accelerating-phi-2/Phi2_Int4_TokenGenerationTP.png'; + import Phi3Image from '../../routes/blogs/accelerating-phi-3/Phi3_Thumbnail.png'; import { createEventDispatcher } from 'svelte'; import ORT117Thumbnail from '../../images/blogs/ort-1-17-thumbnail.png'; import WebGPUImage from '../../images/blogs/webgpu_blog_thumbnail.jpg'; @@ -41,6 +42,16 @@ dispatch('switchTab', tab); } let featuredblog = [ + { + title: 'ONNX Runtime supports Phi-3 mini models across platforms and devices', + date: 'April 22nd, 2024', + blurb: + "You can now run Microsoft's latest home-grown Phi-3 models across a huge range of devices and platforms thanks to ONNX Runtime and DirectML.", + link: 'blogs/accelerating-phi-3', + image: Phi3Image, + imgalt: + 'Phi-3 + ONNX Runtime with the prompt "Tell me a joke" and Phi-3 answering: "Why don\'t scientists trust atoms?" "Because they make up everything!"' + }, { title: 'ONNX Runtime Web unleashes generative AI in the browser using WebGPU', date: 'February 29th, 2024', @@ -60,6 +71,9 @@ image: ORT117Thumbnail, imgalt: 'ONNX Runtime 1.17 release logo' }, + + ]; + let blogs = [ { title: 'Accelerating Phi-2, CodeLlama, Gemma and other Gen AI models with ONNX Runtime', date: 'February 26th, 2024', @@ -67,9 +81,7 @@ link: 'blogs/accelerating-phi-2', image: Phi2Image, imgalt: 'Phi2 float16 token generation throughput comparison' - } - ]; - let blogs = [ + }, { title: 'On-Device Training: Training a model in browser', date: 'February 6th, 2024', diff --git a/src/routes/blogs/accelerating-phi-3/+page.svx b/src/routes/blogs/accelerating-phi-3/+page.svx new file mode 100644 index 0000000000..8d65a47974 --- /dev/null +++ b/src/routes/blogs/accelerating-phi-3/+page.svx @@ -0,0 +1,143 @@ +--- +title: ONNX Runtime supports Phi-3 mini models across platforms and devices +date: '22nd April, 2024' +description: 'Thanks to day one ONNX Runtime and DirectML support, developers can now deploy Phi-3 Mini at scale' +keywords: 'GenAI , LLM, ONNXRuntime, ORT, Phi, DirectML, Windows' +authors: + [ + ] +authorsLink: + [ + ] +image: 'Phi3_Thumbnail.png' +url: 'https://onnxruntime.ai/blogs/accelerating-phi-3' +--- + +You can now run Microsoft's latest home-grown [Phi-3 models](https://aka.ms/phi3blog-april) across a huge range of devices and platforms thanks to ONNX Runtime and DirectML. Today we're proud to announce day 1 support for both flavors of Phi-3, [phi3-mini-4k-instruct](https://aka.ms/phi3-mini-4k-instruct) and [phi3-mini-128k-instruct](https://aka.ms/phi3-mini-128k-instruct). The optimized ONNX models are available at [phi3-mini-4k-instruct-onnx](https://aka.ms/phi3-mini-4k-instruct-onnx) and [phi3-mini-128k-instruct-onnx](https://aka.ms/phi3-mini-128k-instruct-onnx). + +Many language models are too large to run locally on most devices, but Phi-3 represents a significant exception to this rule: this small but mighty suite of models achieves comparable performance to models 10 times larger! Phi-3 Mini is also the first model in its weight class to support long contexts of up to 128K tokens. To learn more about how Microsoft's strategic data curation and innovative scaling achieved these remarkable results, see [here](https://aka.ms/phi3-tech-report). + +You can easily get started with Phi-3 with our newly introduced ONNX runtime Generate() API, found [here](https://github.com/microsoft/onnxruntime-genai/blob/main/examples/python/phi-3-tutorial.md)! + +## DirectML and ONNX Runtime scales Phi-3 Mini on Windows + +By itself, Phi-3 is already small enough to run on many Windows devices, but why stop there? Making Phi-3 even smaller with quantization would dramatically expand the model's reach on Windows, but not all quantization techniques are created equal. We wanted to ensure scalability while also maintaining model accuracy. + +Activation-Aware Quantization (AWQ) to quantize Phi-3 Mini lets us reap the memory savings from quantization with only a minimal impact on accuracy. AWQ achieves this by identifying the top 1% of salient weights that are necessary for maintaining model accuracy and quantizing the remaining 99% of weights. This leads to much less accuracy loss from quantization with AWQ compared to many other quantization techniques. For more on AWQ, see [here](https://arxiv.org/abs/2306.00978). + +Every GPU that supports DirectX 12 on Windows can run DirectML, regardless of whether it's an AMD, Intel, or NVIDIA GPU. DirectML and ONNX Runtime now support INT4 AWQ, which means developers can now run and deploy this quantized version of Phi-3 across hundreds of millions of Windows devices! + +We're working with our hardware vendor partners to provide driver updates that will further improve performance in the coming weeks. Attend our [Build Talk](https://build.microsoft.com/en-US/sessions/65c11f47-56d8-442b-ae52-48df62b7b542?source=sessions) in late May to learn more! + +See below for dedicated performance numbers. + +## ONNX Runtime for Mobile + +In addition to supporting both Phi-3 Mini models on various GPUs, ONNX Runtime can help run these models on Mobile, Windows, and Mac CPUs, making it a truly cross-platform framework. ONNX Runtime also supports quantization techniques like RTN to enable these models to run across many different hardware. + +ONNX Runtime Mobile empowers developers to perform on-device inference with AI models on mobile and edge devices. By removing client-server communications, ORT Mobile provides privacy protection and has zero cost. Using RTN INT4 quantization, we significantly reduce the size of the state-of-the-art Phi-3 Mini models and can run both on a Samsung Galaxy S21 at a moderate speed. When applying RTN INT4 quantization, there is a tuning parameter for the int4 accuracy level. This parameter specifies the minimum accuracy level required for the activation of MatMul in int4 quantization, balancing performance and accuracy trade-offs. Two versions of RTN quantized models have been released with int4_accuracy_level=1, optimized for accuracy, and int4_accuracy_level=4, optimized for performance. If you prefer better performance with a slight trade-off in accuracy, we recommend using the model with int4_accuracy_level=4. + + +## ONNX Runtime for Server Scenarios + +For Linux developers and beyond, ONNX Runtime with CUDA is a great solution that supports a wide range of NVIDIA GPUs, including both consumer and data center GPUs. Phi-3 Mini-128K-Instruct performs better for ONNX Runtime with CUDA than PyTorch for all batch size, prompt length combinations. + +For FP16 CUDA and INT4 CUDA, Phi-3 Mini-128K-Instruct with ORT performs up to 5X faster and up to 9X faster than PyTorch, respectively. Phi-3 Mini-128K-Instruct is currently not supported by Llama.cpp. + +For FP16 and INT4 CUDA, Phi-3 Mini-4K-Instruct with ORT performs up to 5X faster and up to 10X faster than PyTorch, respectively. Phi-3 Mini-4K-Instruct is also up to 3X faster than Llama.cpp for large sequence lengths. + +In addition to supporting both Phi-3 Mini models on various GPUs, ONNX Runtime can help run these models on mobile, Windows, and Mac CPUs, making it a truly cross-platform framework. ONNX Runtime also supports quantization techniques like RTN to enable these models to run across many different hardware. + +ONNX Runtime Mobile empowers developers to perform on-device inference with AI models on mobile and edge devices. By removing client-server communications, ORT Mobile provides privacy protection and has zero cost. Using RTN INT4 quantization, we significantly reduce the size of the state-of-the-art Phi-3 Mini models and can run both on a Samsung Galaxy S21 at a moderate speed. When applying RTN INT4 quantization, there is a tuning parameter for the INT4 accuracy level. This parameter specifies the minimum accuracy level required for the activation of MatMul in INT4 quantization, balancing performance and accuracy trade-offs. Two versions of RTN quantized models have been released: (1) the model optimized for accuracy with int4_accuracy_level=1 and (2) the model optimized for performance with int4_accuracy_level=4. If you prefer better performance with a slight trade-off in accuracy, we recommend using the model with int4_accuracy_level=4. + +Whether it's Windows, Linux, Android, or Mac, there's a path to infer models efficiently with ONNX Runtime! + +## Try the ONNX Runtime Generate() API + +We are pleased to announce our new Generate() API, which makes it easier to run the Phi-3 models across a range of devices, platforms, and EP backends by wrapping several aspects of generative AI inferencing. The Generate() API makes it easy to drag and drop LLMs straight into your app. To run the early version of these models with ONNX, follow the steps [here](http://aka.ms/generate-tutorial). + + +Example: +
+
+python model-qa.py -m /YourModelPath/onnx/cpu_and_mobile/phi-3-mini-4k-instruct-int4-cpu -k 40 -p 0.95 -t 0.8 -r 1.0
+
+Input: <user> Tell me a joke <end>
+
+Output: <assistant> Why don't scientists trust atoms?
+
+Because they make up everything!
+
+This joke plays on the double meaning of "make up." In science, atoms are the fundamental building blocks of matter,
+literally making up everything. However, in a colloquial sense, "to make up" can mean to fabricate or lie, hence the humor. <end>
+
+
+Please watch this space for more updates on AMD, and additional optimization with ORT 1.18. Also, check out our [Build Talk](https://build.microsoft.com/en-US/sessions/e6d21a49-2efb-4a39-8c26-f6eef1410c7a?source=sessions) in late May to learn more about this API! + +## Performance Metrics + +### DirectML: +
+
+We measured ONNX Runtime + DirectML performance of Phi-3 Mini (4k sequence length) quantized with AWQ and with a block size of 128 on Windows. Our test machine had an NVIDIA GeForce RTX 4090 GPU and an Intel Core i9-13900K CPU. As you can see in the table, DirectML offers high token throughput even at longer prompts and generation lengths. +

+DirectML lets developers not only achieve great performance but also deploy models across the entire Windows ecosystem with support from AMD, Intel and NVIDIA. Best of all, AWQ means that developers get this scale while also maintaining high model accuracy. +

+Stay tuned for additional performance improvements in the coming weeks thanks to optimized drivers from our hardware partners and additional updates to the ONNX Generate() API. + +
+
+ + + + + + + + + + + + + + + +Results computed with Batch size = 1 +
Prompt LengthGeneration LengthWall Clock tokens/s
16256266.65
16512251.63
161024238.87
162048217.5
32256278.53
32512259.73
321024241.72
322048219.3
64256308.26
64512272.47
641024245.67
642048220.55
+ +
+
+ +### CUDA: + +The table below shows improvement on the average throughput of the first 256 tokens generated (tps) for the Phi-3 Mini 128K Instruct ONNX model. The comparisons are for FP16 and INT4 precisions on CUDA, as measured on 1 A100 80GB GPU (SKU: [Standard_ND96amsr_A100_v4](https://learn.microsoft.com/en-us/azure/virtual-machines/ndm-a100-v4-series)). + +
+Average throughput of Phi-3 Mini 128K Instruct ONNX model. + +Note: PyTorch Compile and Llama.cpp do not currently support the Phi-3 Mini 128K instruct model. +
+
+ + +The table below shows improvement on the average throughput of the first 256 tokens generated (tps) for Phi-3 Mini 4K Instruct ONNX model. The comparisons are for FP16 and INT4 precisions on CUDA, as measured on 1 A100 80GB GPU (SKU: [Standard_ND96amsr_A100_v4](https://learn.microsoft.com/en-us/azure/virtual-machines/ndm-a100-v4-series)). +
+Average throughput of int4 Phi-3 Mini 4K Instruct ONNX model. + +Average throughput of fp16 Phi-3 Mini 4K Instruct ONNX model. + +
+Pip packages: torch 2.2.0, triton 2.2.0, onnxruntime-gpu1.18.0, transformers 4.39.0, bitsandbytes 0.42.0 +
+
+Performance is improved across CPU and other devices as well. + +## Safety + +Safety metrics and RAI align with the base Phi-3 models. See [here](https://aka.ms/phi3blog-april) for more details. + +## Try ONNX Runtime for Phi3 + +This blog post introduces how ONNX Runtime and DirectML optimize the Phi-3 model. We've included instructions for running Phi-3 across Windows and other platforms, as well as early benchmarking results. Further improvements and perf optimizations are under way, so stay tuned for the ONNX Runtime 1.18 [release](https://github.com/microsoft/onnxruntime/releases) in early May! + +We encourage you to try out Phi-3 and share your feedback in the [ONNX Runtime](https://github.com/microsoft/onnxruntime) GitHub repository! \ No newline at end of file diff --git a/src/routes/blogs/accelerating-phi-3/Phi3-4k-FP16CUDA.png b/src/routes/blogs/accelerating-phi-3/Phi3-4k-FP16CUDA.png new file mode 100644 index 0000000000..bb243d9ab5 Binary files /dev/null and b/src/routes/blogs/accelerating-phi-3/Phi3-4k-FP16CUDA.png differ diff --git a/src/routes/blogs/accelerating-phi-3/Phi3-4k-Int4CUDA.png b/src/routes/blogs/accelerating-phi-3/Phi3-4k-Int4CUDA.png new file mode 100644 index 0000000000..c5f7bca42f Binary files /dev/null and b/src/routes/blogs/accelerating-phi-3/Phi3-4k-Int4CUDA.png differ diff --git a/src/routes/blogs/accelerating-phi-3/Phi3-Mini-128K-FP16CUDA.png b/src/routes/blogs/accelerating-phi-3/Phi3-Mini-128K-FP16CUDA.png new file mode 100644 index 0000000000..4c82284405 Binary files /dev/null and b/src/routes/blogs/accelerating-phi-3/Phi3-Mini-128K-FP16CUDA.png differ diff --git a/src/routes/blogs/accelerating-phi-3/Phi3_Thumbnail.png b/src/routes/blogs/accelerating-phi-3/Phi3_Thumbnail.png new file mode 100644 index 0000000000..4209c0587c Binary files /dev/null and b/src/routes/blogs/accelerating-phi-3/Phi3_Thumbnail.png differ diff --git a/src/routes/blogs/blog-post-featured.svelte b/src/routes/blogs/blog-post-featured.svelte index 9a394a16f1..b514fe4b29 100644 --- a/src/routes/blogs/blog-post-featured.svelte +++ b/src/routes/blogs/blog-post-featured.svelte @@ -32,7 +32,7 @@

{title}

{description}

- {imgalt} + {imgalt}
{date}
diff --git a/src/routes/blogs/github-markdown-light.css b/src/routes/blogs/github-markdown-light.css index 1776b4cd0f..9da971ef1b 100644 --- a/src/routes/blogs/github-markdown-light.css +++ b/src/routes/blogs/github-markdown-light.css @@ -6,6 +6,11 @@ ul { width: 35em; } /*light*/ +@media (max-width: 768px) { + .w50 { + width: 75%; + } + } .markdown-body { -ms-text-size-adjust: 100%; diff --git a/src/routes/blogs/post.svelte b/src/routes/blogs/post.svelte index 2d93cbf875..1018bdfc73 100644 --- a/src/routes/blogs/post.svelte +++ b/src/routes/blogs/post.svelte @@ -68,7 +68,11 @@

{title}

- By: + {#if authors.length === 0} +
+ {:else} +

By:

+ {/if} {#each authors as author, i} {author}{i + 1 === authors.length ? ''