diff --git a/onnxruntime/core/mlas/lib/q4_dq.cpp b/onnxruntime/core/mlas/lib/q4_dq.cpp
index acc3cdd651..39d921a76f 100644
--- a/onnxruntime/core/mlas/lib/q4_dq.cpp
+++ b/onnxruntime/core/mlas/lib/q4_dq.cpp
@@ -481,7 +481,11 @@ struct BlockwiseQuantizer {
             thread_pool, total_thrd_blks,
             [&](ptrdiff_t block_idx) {
                 uint8_t zp_bytes[BitsTraits<qbits, false>::kPackSize];
-                std::fill_n(zp_bytes, BitsTraits<qbits, false>::kPackSize, (uint8_t)(BitsTraits<qbits, false>::kMid));
+                if constexpr (qbits == 2)
+                  std::fill_n(zp_bytes, BitsTraits<qbits, false>::kPackSize, (uint8_t)2);
+                if constexpr (qbits == 4)
+                  std::fill_n(zp_bytes, BitsTraits<qbits, false>::kPackSize, (uint8_t)8);
+
 
                 const int32_t r_blk_idx = static_cast<int32_t>(block_idx / thrd_col_blks);
                 const int32_t c_blk_idx = static_cast<int32_t>(block_idx % thrd_col_blks);
@@ -524,14 +528,13 @@ struct BlockwiseQuantizer {
 
                 // !! qbits specific code as we need to pack 2 4b numbers into one byte
                 if (zero_points != nullptr) {
-                  const int32_t meta_idx = meta_col * ((row_blks + 1) / BitsTraits<qbits, false>::kPackSize) + meta_row / BitsTraits<qbits, false>::kPackSize;
                   if constexpr (qbits == 4) {
+                    const int32_t meta_idx = meta_col * ((row_blks + 1) / BitsTraits<qbits, false>::kPackSize) + meta_row / BitsTraits<qbits, false>::kPackSize;
                     zero_points[meta_idx] = (zp_bytes[0] & 0xf) | (zp_bytes[1] << 4);
                   } else if constexpr (qbits == 2) {
+                      const int32_t meta_idx = meta_col * ((row_blks + 3) / BitsTraits<qbits, false>::kPackSize) + meta_row / BitsTraits<qbits, false>::kPackSize;
                     zero_points[meta_idx] = (zp_bytes[0] & 0x3) | ((zp_bytes[1] & 0x3) << 2) |
                       ((zp_bytes[2] & 0x3) << 4) | ((zp_bytes[3] & 0x3) << 6);
-                  } else {
-                      static_assert(false && "only support qbits of 4 and 2");
                   }
                 }
 
@@ -670,7 +673,8 @@ struct BlockwiseQuantizer {
                                 dst[j * rows + (i + 1)] = static_cast<ElementT>(v1);
                             }
                         } else {
-                            const int zp_quad = zero_points[meta_col * ((row_blks + 3) / pack_size) + meta_row / pack_size];
+                            const int zp_quad = (zero_points == nullptr) ?
+                              0xAA : zero_points[meta_col * ((row_blks + 3) / pack_size) + meta_row / pack_size];
                             int zp = 0;
                             const int meta_row_mod = meta_row % 4;
                             switch (meta_row_mod) {
@@ -730,19 +734,35 @@ struct BlockwiseQuantizer {
  * @tparam signed_quant  quantized type is signed
  */
 template <typename Tin, int qbits, bool signed_quant>
-struct BlockwiseQDQQuantizer;
-
-template <typename Tin, bool signed_quant>
-struct BlockwiseQDQQuantizer<Tin, 4, signed_quant> {
+struct BlockwiseQDQQuantizer {
     static MLAS_FORCEINLINE uint8_t GetElem(uint8_t val, int32_t idx)
     {
-        return (val >> (idx << 2)) & 0xF;
+        if constexpr (qbits == 2) {
+            return (val >> (idx << 1)) & 0x3;
+        } else if constexpr (qbits == 4) {
+            return (val >> (idx << 2)) & 0xF;
+        }
     }
 
     static MLAS_FORCEINLINE uint8_t SetElem(uint8_t val, int32_t idx, uint8_t dst)
     {
-        auto shift = idx << 2;
-        return ((val & 0xF) << shift) | (dst & (~(0xF << shift)));
+        if constexpr (qbits == 2) {
+            auto shift = idx << 1;
+            return ((val & 0x3) << shift) | (dst & (~(0x3 << shift)));
+        } else if constexpr (qbits == 4) {
+            auto shift = idx << 2;
+            return ((val & 0xF) << shift) | (dst & (~(0xF << shift)));
+        }
+    }
+
+    template <bool add2>
+    static MLAS_FORCEINLINE uint8_t Pack(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3)
+    {
+          if constexpr (add2) {
+            return ((v0 & 0x3) ^ 2) | (((v1 & 0x3) ^ 2) << 2) | (((v2 & 0x3) ^ 2) << 4) | (((v3 & 0x3) ^ 2) << 6);
+          } else {
+              return (v0 & 0x3) | ((v1 & 0x3) << 2) | ((v2 & 0x3) << 4) | ((v3 & 0x3) << 6);
+          }
     }
 
     template <bool add8>
@@ -1491,7 +1511,7 @@ MlasBlockwiseQuantizedShape(
 
 template
 void
-MlasBlockwiseQuantMetaShape<float, 4>(
+MlasBlockwiseQuantMetaShape<float, 2>(
     int block_size,
     bool columnwise,
     int rows,
@@ -1500,6 +1520,16 @@ MlasBlockwiseQuantMetaShape<float, 4>(
     int& meta_cols
     );
 
+template void
+MlasBlockwiseQuantMetaShape<float, 4>(
+    int block_size,
+    bool columnwise,
+    int rows,
+    int columns,
+    int& meta_rows,
+    int& meta_cols
+);
+
 template
 void
 MlasBlockwiseQuantMetaShape<MLAS_FP16, 4>(
@@ -1901,6 +1931,19 @@ MlasQDQQuantizeBlockwise<float, 4>(
     MLAS_THREADPOOL* thread_pool
 );
 
+template bool
+MlasQDQQuantizeBlockwise<float, 2>(
+    const float* src,
+    float* scales,
+    uint8_t* zero_points,
+    uint8_t* dst,
+    bool columnwise,
+    int rows,
+    int columns,
+    int quant_block_size,
+    MLAS_THREADPOOL* thread_pool
+);
+
 template bool
 MlasQDQQuantizeBlockwise<MLAS_FP16, 4>(
     const MLAS_FP16* src,
@@ -1940,6 +1983,36 @@ MlasQDQTransposeBlockwiseQuantized(
     }
 }
 
+template void
+MlasQDQTransposeBlockwiseQuantized<float, 2, true>(
+    const uint8_t* src_weights,
+    const float* src_scales,
+    const uint8_t* src_zero_points,
+    uint8_t* dst_weights,
+    float* dst_scales,
+    uint8_t* dst_zero_points,
+    bool columnwise,
+    int rows,
+    int columns,
+    int quant_block_size,
+    MLAS_THREADPOOL* thread_pool
+);
+
+template void
+MlasQDQTransposeBlockwiseQuantized<float, 2, false>(
+    const uint8_t* src_weights,
+    const float* src_scales,
+    const uint8_t* src_zero_points,
+    uint8_t* dst_weights,
+    float* dst_scales,
+    uint8_t* dst_zero_points,
+    bool columnwise,
+    int rows,
+    int columns,
+    int quant_block_size,
+    MLAS_THREADPOOL* thread_pool
+);
+
 template void
 MlasQDQTransposeBlockwiseQuantized<float, 4, true>(
     const uint8_t* src_weights,
diff --git a/onnxruntime/test/mlas/unittest/test_blockq4.cpp b/onnxruntime/test/mlas/unittest/test_blockq4.cpp
index 11e5cec1f1..fbe9e8b5f0 100644
--- a/onnxruntime/test/mlas/unittest/test_blockq4.cpp
+++ b/onnxruntime/test/mlas/unittest/test_blockq4.cpp
@@ -19,6 +19,9 @@ Abstract:
 #include "test_util.h"
 #include "mlas_q4.h"
 
+constexpr int Q2Bits = 2;
+constexpr int Q4Bits = 4;
+
 class MlasBlockwiseQdqTest : public MlasTestBase {
  private:
   MatrixGuardBuffer<float> FpBuf;
@@ -36,6 +39,7 @@ class MlasBlockwiseQdqTest : public MlasTestBase {
   MatrixGuardBuffer<float> QDQTransposedOutputScales;
   MatrixGuardBuffer<uint8_t> QDQTransposedOutputOffsets;
 
+  template<int qbits>
   void Test(int rows, int columns, int block_size, bool columnwise, bool symmetric) {
     float* dequant_buf = FpBuf.GetBuffer(rows * columns, true);
     float* transposed = FpBuf2.GetBuffer(rows * columns, true);
@@ -46,41 +50,79 @@ class MlasBlockwiseQdqTest : public MlasTestBase {
 
     int meta_rows;
     int meta_cols;
-    MlasBlockwiseQuantMetaShape<float, 4>(block_size, columnwise, rows, columns, meta_rows, meta_cols);
+    MlasBlockwiseQuantMetaShape<float, qbits>(block_size, columnwise, rows, columns, meta_rows, meta_cols);
 
     int q_rows;
     int q_cols;
-    MlasBlockwiseQuantizedShape<float, 4>(block_size, columnwise, rows, columns, q_rows, q_cols);
+    MlasBlockwiseQuantizedShape<float, qbits>(block_size, columnwise, rows, columns, q_rows, q_cols);
 
     size_t q_data_size_in_bytes, q_scale_size, q_zp_size_in_bytes;
-    MlasBlockwiseQuantizedBufferSizes<4>(block_size, columnwise, rows, columns,
+    MlasBlockwiseQuantizedBufferSizes<qbits>(block_size, columnwise, rows, columns,
                                       q_data_size_in_bytes, q_scale_size, &q_zp_size_in_bytes);
 
     uint8_t* elements = InputElements.GetBuffer(q_data_size_in_bytes, true);
     uint8_t* qdq_weights = QDQOutputElements.GetBuffer((rows * columns + 1) / 2, true);
     uint8_t* qdq_weights_T = QDQTransposedOutputElements.GetBuffer(q_data_size_in_bytes, true);
 
-    int v = 7;
-    for (int c = 0; c < columns; c++) {
-      for (int r = 0; r < rows; r += 2) {
-        int idx = c * q_rows + r / 2;
-        uint8_t v0 = static_cast<uint8_t>(v);
-        v = (v + 5) % 16;
-        if (v == 11 || v == 7 || v == 3) {
-          // making the cycle 13 instead of 16, avoiding same values in a row
-          v = (v + 5) % 16;
+    int pack_size = 8 / qbits;
+    int v;
+    if constexpr (qbits == 2) {
+      v = 1;
+      for (int c = 0; c < columns; c++) {
+        for (int r = 0; r < rows; r += pack_size) {
+          int idx = c * q_rows + r / pack_size;
+          uint8_t v0 = static_cast<uint8_t>(v);
+          v = (v + 1) % 4;
+          uint8_t v1 = 0;
+          if (r + 1 < rows) {
+            v1 = static_cast<uint8_t>(v);
+            v = (v + 1) % 4;
+            if (v == 3) {
+              v = (v + 1) % 4;
+            }
+          }
+          uint8_t v2 = 0;
+          if (r + 2 < rows) {
+            v2 = static_cast<uint8_t>(v);
+            v = (v + 1) % 4;
+            if (v == 3) {
+              v = (v + 1) % 4;
+            }
+          }
+          uint8_t v3 = 0;
+          if (r + 3 < rows) {
+            v3 = static_cast<uint8_t>(v);
+            v = (v + 1) % 4;
+            if (v == 3) {
+              v = (v + 1) % 4;
+            }
+          }
+          elements[idx] = (v3 << 6) | (v2 << 4) | (v1 << 2) | v0;
         }
-        uint8_t v1 = 0;
-        if (r + 1 < rows) {
-          v1 = static_cast<uint8_t>(v);
+      }
+    } else if constexpr(qbits == 4) {
+      v = 7;
+      for (int c = 0; c < columns; c++) {
+        for (int r = 0; r < rows; r += 2) {
+          int idx = c * q_rows + r / 2;
+          uint8_t v0 = static_cast<uint8_t>(v);
           v = (v + 5) % 16;
           if (v == 11 || v == 7 || v == 3) {
             // making the cycle 13 instead of 16, avoiding same values in a row
             v = (v + 5) % 16;
           }
-        }
+          uint8_t v1 = 0;
+          if (r + 1 < rows) {
+            v1 = static_cast<uint8_t>(v);
+            v = (v + 5) % 16;
+            if (v == 11 || v == 7 || v == 3) {
+              // making the cycle 13 instead of 16, avoiding same values in a row
+              v = (v + 5) % 16;
+            }
+          }
 
-        elements[idx] = (v1 << 4) | v0;
+          elements[idx] = (v1 << 4) | v0;
+        }
       }
     }
 
@@ -91,30 +133,57 @@ class MlasBlockwiseQdqTest : public MlasTestBase {
     uint8_t* qdq_zp = symmetric ? nullptr : QDQOutputOffsets.GetBuffer(zp_size, true);
     uint8_t* qdq_zp_T = symmetric ? nullptr : QDQTransposedOutputOffsets.GetBuffer(q_zp_size_in_bytes, true);
     if (zp) {
-      for (int c = 0; c < meta_cols; c++) {
-        for (int r = 0; r < meta_rows; r += 2) {
-          int idx = c * ((meta_rows + 1) / 2) + r / 2;
-          uint8_t v0 = static_cast<uint8_t>(v);
-          v = (v + 5) % 16;
-          if (v == 11 || v == 7 || v == 3) {
-            // making the cycle 13 instead of 16, avoiding same values in a row
-            v = (v + 5) % 16;
+      if constexpr (qbits == 2) {
+        for (int c = 0; c < meta_cols; c++) {
+          for (int r = 0; r < meta_rows; r += pack_size) {
+            int idx = c * ((meta_rows + 3) / pack_size) + r / pack_size;
+            uint8_t v0 = static_cast<uint8_t>(v);
+            v = (v + 1) % 4;
+            uint8_t v1 = 0;
+            if (r + 1 < meta_rows) {
+              v1 = static_cast<uint8_t>(v);
+              v = (v + 1) % 4;
+            }
+            uint8_t v2 = 0;
+            if (r + 2 < meta_rows) {
+              v2 = static_cast<uint8_t>(v);
+              v = (v + 1) % 4;
+            }
+            uint8_t v3 = 0;
+            if (r + 3 < meta_rows) {
+              v3 = static_cast<uint8_t>(v);
+              v = (v + 1) % 4;
+            }
+            zp[idx] = (v3 << 6) | (v2 << 4) | (v1 << 2) | v0;
           }
-          uint8_t v1 = 0;
-          if (r + 1 < meta_rows) {
-            v1 = static_cast<uint8_t>(v);
+        }
+      }
+      else if constexpr (qbits == 4) {
+        for (int c = 0; c < meta_cols; c++) {
+          for (int r = 0; r < meta_rows; r += 2) {
+            int idx = c * ((meta_rows + 1) / 2) + r / 2;
+            uint8_t v0 = static_cast<uint8_t>(v);
             v = (v + 5) % 16;
             if (v == 11 || v == 7 || v == 3) {
               // making the cycle 13 instead of 16, avoiding same values in a row
               v = (v + 5) % 16;
             }
+            uint8_t v1 = 0;
+            if (r + 1 < meta_rows) {
+              v1 = static_cast<uint8_t>(v);
+              v = (v + 5) % 16;
+              if (v == 11 || v == 7 || v == 3) {
+                // making the cycle 13 instead of 16, avoiding same values in a row
+                v = (v + 5) % 16;
+              }
+            }
+            zp[idx] = (v1 << 4) | v0;
           }
-          zp[idx] = (v1 << 4) | v0;
         }
       }
     }
 
-    MlasDequantizeBlockwise<float, 4>(dequant_buf, elements, scales, zp, block_size,
+    MlasDequantizeBlockwise<float, qbits>(dequant_buf, elements, scales, zp, block_size,
                                       columnwise, rows, columns, threadpool_ptr);
 
     MlasTranspose(dequant_buf, transposed, columns, rows);
@@ -123,49 +192,80 @@ class MlasBlockwiseQdqTest : public MlasTestBase {
     float* o_scales = OutputScales.GetBuffer(meta_rows * meta_cols);
     uint8_t* o_zp = symmetric ? nullptr : OutputOffsets.GetBuffer(((meta_rows + 1) / 2) * meta_cols, true);
 
-    MlasQuantizeBlockwise<float, 4>(o_elements, o_scales, o_zp, transposed, block_size,
+    MlasQuantizeBlockwise<float, qbits>(o_elements, o_scales, o_zp, transposed, block_size,
                                     columnwise, rows, columns, columns, threadpool_ptr);
 
-    if (columnwise) {
-      bool signed_quant = MlasQDQQuantizeBlockwise<float, 4>(
-          transposed, qdq_scales, qdq_zp, qdq_weights,
-          true, rows, columns, block_size, threadpool_ptr);
-
-      ASSERT_EQ(symmetric, signed_quant) << "symmetric quantization should be signed";
-
-      if (symmetric) {
-        MlasQDQTransposeBlockwiseQuantized<float, 4, true>(
-            qdq_weights, qdq_scales, qdq_zp, qdq_weights_T, qdq_scales_T, qdq_zp_T,
+    if constexpr (qbits == 4) {
+      if (columnwise) {
+        bool signed_quant = MlasQDQQuantizeBlockwise<float, qbits>(
+            transposed, qdq_scales, qdq_zp, qdq_weights,
             true, rows, columns, block_size, threadpool_ptr);
 
-      } else {
-        MlasQDQTransposeBlockwiseQuantized<float, 4, false>(
-            qdq_weights, qdq_scales, qdq_zp, qdq_weights_T, qdq_scales_T, qdq_zp_T,
-            true, rows, columns, block_size, threadpool_ptr);
+        ASSERT_EQ(symmetric, signed_quant) << "symmetric quantization should be signed";
+
+        if (symmetric) {
+          MlasQDQTransposeBlockwiseQuantized<float, qbits, true>(
+              qdq_weights, qdq_scales, qdq_zp, qdq_weights_T, qdq_scales_T, qdq_zp_T,
+              true, rows, columns, block_size, threadpool_ptr);
+
+        } else {
+          MlasQDQTransposeBlockwiseQuantized<float, qbits, false>(
+              qdq_weights, qdq_scales, qdq_zp, qdq_weights_T, qdq_scales_T, qdq_zp_T,
+              true, rows, columns, block_size, threadpool_ptr);
+        }
       }
     }
 
-    for (int c = 0; c < columns; c++) {
-      for (int r = 0; r < rows; r += 2) {
-        int idx = c * q_rows + r / 2;
-        ASSERT_EQ(o_elements[idx] & 0xf, elements[idx] & 0xf)
-            << ", index=[" << r << "x" << c << "], shape=[" << rows << "x" << columns
-            << "] block: " << block_size << ", symmetric: " << symmetric << ", columnwise: " << columnwise;
-        if (columnwise) {
-          ASSERT_EQ(qdq_weights_T[idx] & 0xf, elements[idx] & 0xf)
+    if constexpr (qbits == 2) {
+      for (int c = 0; c < columns; c++) {
+        for (int r = 0; r < rows; r += pack_size) {
+          int idx = c * q_rows + r / pack_size;
+          ASSERT_EQ(o_elements[idx] & 0x3, elements[idx] & 0x3)
               << ", index=[" << r << "x" << c << "], shape=[" << rows << "x" << columns
               << "] block: " << block_size << ", symmetric: " << symmetric << ", columnwise: " << columnwise;
-        }
-
-        if (r + 1 < rows) {
-          ASSERT_EQ(o_elements[idx] >> 4, elements[idx] >> 4)
-              << ", index=[" << r + 1 << "x" << c << "], shape=[" << rows << "x" << columns
-              << "] block: " << block_size << ", symmetric: " << symmetric << ", columnwise: " << columnwise;
-          if (columnwise) {
-            ASSERT_EQ(qdq_weights_T[idx] >> 4, elements[idx] >> 4)
+          if (r + 1 < rows) {
+            ASSERT_EQ((o_elements[idx] >> 2) & 0x3, (elements[idx] >> 2) & 0x3)
                 << ", index=[" << r + 1 << "x" << c << "], shape=[" << rows << "x" << columns
                 << "] block: " << block_size << ", symmetric: " << symmetric << ", columnwise: " << columnwise;
           }
+          if (r + 2 < rows) {
+            ASSERT_EQ((o_elements[idx] >> 4) & 0x3, (elements[idx] >> 4) & 0x3)
+                << ", index=[" << r + 2 << "x" << c << "], shape=[" << rows << "x" << columns
+                << "] block: " << block_size << ", symmetric: " << symmetric << ", columnwise: " << columnwise;
+          }
+          if (r + 3 < rows) {
+            ASSERT_EQ((o_elements[idx] >> 6) & 0x3, (elements[idx] >> 6) & 0x3)
+                << ", index=[" << r + 3 << "x" << c << "], shape=[" << rows << "x" << columns
+                << "] block: " << block_size << ", symmetric: " << symmetric << ", columnwise: " << columnwise;
+          }
+        }
+      }
+    } else if constexpr (qbits == 4) {
+      for (int c = 0; c < columns; c++) {
+        for (int r = 0; r < rows; r += 2) {
+          int idx = c * q_rows + r / 2;
+          ASSERT_EQ(o_elements[idx] & 0xf, elements[idx] & 0xf)
+              << ", index=[" << r << "x" << c << "], shape=[" << rows << "x" << columns
+              << "] block: " << block_size << ", symmetric: " << symmetric << ", columnwise: " << columnwise;
+          if constexpr (qbits == 4) {
+            if (columnwise) {
+              ASSERT_EQ(qdq_weights_T[idx] & 0xf, elements[idx] & 0xf)
+                  << ", index=[" << r << "x" << c << "], shape=[" << rows << "x" << columns
+                  << "] block: " << block_size << ", symmetric: " << symmetric << ", columnwise: " << columnwise;
+            }
+          }
+          if (r + 1 < rows) {
+            ASSERT_EQ(o_elements[idx] >> 4, elements[idx] >> 4)
+                << ", index=[" << r + 1 << "x" << c << "], shape=[" << rows << "x" << columns
+                << "] block: " << block_size << ", symmetric: " << symmetric << ", columnwise: " << columnwise;
+            if constexpr (qbits == 4) {
+              if (columnwise) {
+                ASSERT_EQ(qdq_weights_T[idx] >> 4, elements[idx] >> 4)
+                    << ", index=[" << r + 1 << "x" << c << "], shape=[" << rows << "x" << columns
+                    << "] block: " << block_size << ", symmetric: " << symmetric << ", columnwise: " << columnwise;
+              }
+            }
+          }
         }
       }
     }
@@ -177,35 +277,64 @@ class MlasBlockwiseQdqTest : public MlasTestBase {
             << ", index=" << r << "x" << c << ", shape=[" << rows << "x" << columns
             << "] block: " << block_size << ", symmetric: " << symmetric << ", columnwise: " << columnwise;
 
-        if (columnwise) {
-          ASSERT_EQ(qdq_scales_T[idx], scales[idx])
-              << ", index=" << r << "x" << c << ", shape=[" << rows << "x" << columns
-              << "] block: " << block_size << ", symmetric: " << symmetric << ", columnwise: " << columnwise;
+        if constexpr (qbits == 4) {
+          if (columnwise) {
+            ASSERT_EQ(qdq_scales_T[idx], scales[idx])
+                << ", index=" << r << "x" << c << ", shape=[" << rows << "x" << columns
+                << "] block: " << block_size << ", symmetric: " << symmetric << ", columnwise: " << columnwise;
+          }
         }
       }
     }
 
     if (symmetric) return;
-    for (int c = 0; c < meta_cols; c++) {
-      for (int r = 0; r < meta_rows; r += 2) {
-        int idx = c * ((meta_rows + 1) / 2) + r / 2;
-        ASSERT_EQ(o_zp[idx] & 0xf, zp[idx] & 0xf)
-            << ", index=" << r << "x" << c << ", shape=[" << rows << "x" << columns
-            << "] block: " << block_size << ", symmetric: " << symmetric << ", columnwise: " << columnwise;
-        if (columnwise) {
-          ASSERT_EQ(qdq_zp_T[idx] & 0xf, zp[idx] & 0xf)
+
+    if constexpr (qbits == 2) {
+      for (int c = 0; c < meta_cols; c++) {
+        for (int r = 0; r < meta_rows; r += pack_size) {
+          int idx = c * ((meta_rows + 3) / pack_size) + r / pack_size;
+          ASSERT_EQ(o_zp[idx] & 0x3, zp[idx] & 0x3)
               << ", index=" << r << "x" << c << ", shape=[" << rows << "x" << columns
               << "] block: " << block_size << ", symmetric: " << symmetric << ", columnwise: " << columnwise;
-        }
-        if (r + 1 < meta_rows) {
-          ASSERT_EQ(o_zp[idx] >> 4, zp[idx] >> 4)
-              << ", index=" << r + 1 << "x" << c << ", shape=[" << rows << "x" << columns
-              << "] block: " << block_size << ", symmetric: " << symmetric << ", columnwise: " << columnwise;
-          if (columnwise) {
-            ASSERT_EQ(qdq_zp_T[idx] >> 4, zp[idx] >> 4)
+          if (r + 1 < meta_rows) {
+            ASSERT_EQ((o_zp[idx] >> 2) & 0x3, (zp[idx] >> 2) & 0x3)
                 << ", index=" << r + 1 << "x" << c << ", shape=[" << rows << "x" << columns
                 << "] block: " << block_size << ", symmetric: " << symmetric << ", columnwise: " << columnwise;
           }
+          if (r + 2 < meta_rows) {
+            ASSERT_EQ((o_zp[idx] >> 4) & 0x3, (zp[idx] >> 4) & 0x3)
+                << ", index=" << r + 2 << "x" << c << ", shape=[" << rows << "x" << columns
+                << "] block: " << block_size << ", symmetric: " << symmetric << ", columnwise: " << columnwise;
+          }
+          if (r + 3 < meta_rows) {
+            ASSERT_EQ((o_zp[idx] >> 6) & 0x3, (zp[idx] >> 6) & 0x3)
+                << ", index=" << r + 3 << "x" << c << ", shape=[" << rows << "x" << columns
+                << "] block: " << block_size << ", symmetric: " << symmetric << ", columnwise: " << columnwise;
+          }
+        }
+      }
+    } else if constexpr (qbits == 4) {
+      for (int c = 0; c < meta_cols; c++) {
+        for (int r = 0; r < meta_rows; r += 2) {
+          int idx = c * ((meta_rows + 1) / 2) + r / 2;
+          ASSERT_EQ(o_zp[idx] & 0xf, zp[idx] & 0xf)
+              << ", index=" << r << "x" << c << ", shape=[" << rows << "x" << columns
+              << "] block: " << block_size << ", symmetric: " << symmetric << ", columnwise: " << columnwise;
+          if (columnwise) {
+            ASSERT_EQ(qdq_zp_T[idx] & 0xf, zp[idx] & 0xf)
+                << ", index=" << r << "x" << c << ", shape=[" << rows << "x" << columns
+                << "] block: " << block_size << ", symmetric: " << symmetric << ", columnwise: " << columnwise;
+          }
+          if (r + 1 < meta_rows) {
+            ASSERT_EQ(o_zp[idx] >> 4, zp[idx] >> 4)
+                << ", index=" << r + 1 << "x" << c << ", shape=[" << rows << "x" << columns
+                << "] block: " << block_size << ", symmetric: " << symmetric << ", columnwise: " << columnwise;
+            if (columnwise) {
+              ASSERT_EQ(qdq_zp_T[idx] >> 4, zp[idx] >> 4)
+                  << ", index=" << r + 1 << "x" << c << ", shape=[" << rows << "x" << columns
+                  << "] block: " << block_size << ", symmetric: " << symmetric << ", columnwise: " << columnwise;
+            }
+          }
         }
       }
     }
@@ -217,44 +346,78 @@ class MlasBlockwiseQdqTest : public MlasTestBase {
     return suite_name.c_str();
   }
 
-  void ExecuteShort(void) override {
-    Test(20, 1, 32, true, false);
-    Test(20, 1, 32, true, true);
-    Test(1, 20, 32, false, false);
-    Test(1, 20, 32, false, true);
-    Test(52, 1, 32, true, false);
-    Test(52, 1, 32, true, true);
-    Test(1, 52, 32, false, false);
-    Test(1, 52, 32, false, true);
-    Test(20, 3, 32, true, false);
-    Test(20, 3, 32, true, true);
-    Test(3, 20, 32, false, false);
-    Test(3, 20, 32, false, true);
-    Test(52, 3, 32, true, false);
-    Test(52, 3, 32, true, true);
-    Test(3, 52, 32, false, false);
-    Test(3, 52, 32, false, true);
-    Test(52, 3, 64, true, false);
-    Test(52, 3, 64, true, true);
-    Test(3, 52, 64, false, false);
-    Test(3, 52, 64, false, true);
-    Test(32 * 9 + 17, 41, 32, true, false);
-    Test(32 * 9 + 17, 41, 32, true, true);
-    Test(41, 32 * 9 + 17, 32, false, false);
-    Test(41, 32 * 9 + 17, 32, false, true);
-    Test(32 * 9 + 17, 41, 64, true, false);
-    Test(32 * 9 + 17, 41, 64, true, true);
-    Test(41, 32 * 9 + 17, 64, false, false);
-    Test(41, 32 * 9 + 17, 64, false, true);
-    Test(32 * 15 + 17, 63, 128, true, false);
-    Test(32 * 15 + 17, 63, 128, true, true);
-    Test(63, 32 * 15 + 17, 128, false, false);
-    Test(63, 32 * 15 + 17, 128, false, true);
+  void ExecuteShort(void) {
+    // only support columnwise = true with qbits=2
+    Test<Q2Bits>(20, 1, 32, true, false);
+    Test<Q2Bits>(20, 1, 32, true, true);
+    //Test<Q2Bits>(1, 20, 32, false, false);
+    //Test<Q2Bits>(1, 20, 32, false, true);
+    Test<Q2Bits>(52, 1, 32, true, false);
+    Test<Q2Bits>(52, 1, 32, true, true);
+    //Test<Q2Bits>(1, 52, 32, false, false);
+    //Test<Q2Bits>(1, 52, 32, false, true);
+    Test<Q2Bits>(20, 3, 32, true, false);
+    Test<Q2Bits>(20, 3, 32, true, true);
+    //Test<Q2Bits>(3, 20, 32, false, false);
+    //Test<Q2Bits>(3, 20, 32, false, true);
+    Test<Q2Bits>(52, 3, 32, true, false);
+    Test<Q2Bits>(52, 3, 32, true, true);
+    //Test<Q2Bits>(3, 52, 32, false, false);
+    //Test<Q2Bits>(3, 52, 32, false, true);
+    Test<Q2Bits>(52, 3, 64, true, false);
+    Test<Q2Bits>(52, 3, 64, true, true);
+    //Test<Q2Bits>(3, 52, 64, false, false);
+    //Test<Q2Bits>(3, 52, 64, false, true);
+    Test<Q2Bits>(32 * 9 + 17, 41, 32, true, false);
+    Test<Q2Bits>(32 * 9 + 17, 41, 32, true, true);
+    //Test<Q2Bits>(41, 32 * 9 + 17, 32, false, false);
+    //Test<Q2Bits>(41, 32 * 9 + 17, 32, false, true);
+    Test<Q2Bits>(32 * 9 + 17, 41, 64, true, false);
+    Test<Q2Bits>(32 * 9 + 17, 41, 64, true, true);
+    //Test<Q2Bits>(41, 32 * 9 + 17, 64, false, false);
+    //Test<Q2Bits>(41, 32 * 9 + 17, 64, false, true);
+    Test<Q2Bits>(32 * 15 + 17, 63, 128, true, false);
+    Test<Q2Bits>(32 * 15 + 17, 63, 128, true, true);
+    //Test<Q2Bits>(63, 32 * 15 + 17, 128, false, false);
+    //Test<Q2Bits>(63, 32 * 15 + 17, 128, false, true);
 
-    Test(256, 256, 32, true, false);
-    Test(256, 256, 32, true, true);
-    Test(256, 256, 32, false, false);
-    Test(256, 256, 32, false, true);
+    Test<Q4Bits>(20, 1, 32, true, false);
+    Test<Q4Bits>(20, 1, 32, true, true);
+    Test<Q4Bits>(1, 20, 32, false, false);
+    Test<Q4Bits>(1, 20, 32, false, true);
+    Test<Q4Bits>(52, 1, 32, true, false);
+    Test<Q4Bits>(52, 1, 32, true, true);
+    Test<Q4Bits>(1, 52, 32, false, false);
+    Test<Q4Bits>(1, 52, 32, false, true);
+    Test<Q4Bits>(20, 3, 32, true, false);
+    Test<Q4Bits>(20, 3, 32, true, true);
+    Test<Q4Bits>(3, 20, 32, false, false);
+    Test<Q4Bits>(3, 20, 32, false, true);
+    Test<Q4Bits>(52, 3, 32, true, false);
+    Test<Q4Bits>(52, 3, 32, true, true);
+    Test<Q4Bits>(3, 52, 32, false, false);
+    Test<Q4Bits>(3, 52, 32, false, true);
+    Test<Q4Bits>(52, 3, 64, true, false);
+    Test<Q4Bits>(52, 3, 64, true, true);
+    Test<Q4Bits>(3, 52, 64, false, false);
+    Test<Q4Bits>(3, 52, 64, false, true);
+    Test<Q4Bits>(32 * 9 + 17, 41, 32, true, false);
+    Test<Q4Bits>(32 * 9 + 17, 41, 32, true, true);
+    Test<Q4Bits>(41, 32 * 9 + 17, 32, false, false);
+    Test<Q4Bits>(41, 32 * 9 + 17, 32, false, true);
+    Test<Q4Bits>(32 * 9 + 17, 41, 64, true, false);
+    Test<Q4Bits>(32 * 9 + 17, 41, 64, true, true);
+    Test<Q4Bits>(41, 32 * 9 + 17, 64, false, false);
+    Test<Q4Bits>(41, 32 * 9 + 17, 64, false, true);
+    Test<Q4Bits>(32 * 15 + 17, 63, 128, true, false);
+    Test<Q4Bits>(32 * 15 + 17, 63, 128, true, true);
+    Test<Q4Bits>(63, 32 * 15 + 17, 128, false, false);
+    Test<Q4Bits>(63, 32 * 15 + 17, 128, false, true);
+
+    Test<Q4Bits>(256, 256, 32, true, false);
+    Test<Q4Bits>(256, 256, 32, true, true);
+    Test<Q4Bits>(256, 256, 32, false, false);
+    Test<Q4Bits>(256, 256, 32, false, true);
   }
 
   MlasBlockwiseQdqTest() = default;