diff --git a/.lintrunner.toml b/.lintrunner.toml
index b2fa676f8e1..16357c27f95 100644
--- a/.lintrunner.toml
+++ b/.lintrunner.toml
@@ -738,6 +738,7 @@ init_command = [
     'python3',
     'tools/linter/adapters/pip_init.py',
     '--dry-run={{DRYRUN}}',
+    '--no-black-binary',
     'black==22.3.0',
     'ufmt==1.3.3',
     'usort==1.0.2',
diff --git a/tools/linter/adapters/pip_init.py b/tools/linter/adapters/pip_init.py
index f921bcb73e3..f177a920d0b 100644
--- a/tools/linter/adapters/pip_init.py
+++ b/tools/linter/adapters/pip_init.py
@@ -37,8 +37,8 @@ if __name__ == "__main__":
         "--dry-run", help="do not install anything, just print what would be done."
     )
     parser.add_argument(
-        "--no-binary",
-        help="do not use pre-compiled binaries from pip.",
+        "--no-black-binary",
+        help="do not use pre-compiled binaries from pip for black.",
         action="store_true",
     )
 
@@ -72,7 +72,7 @@ if __name__ == "__main__":
                 "Package {package_name} did not have a version specified. "
                 "Please specify a version to produce a consistent linting experience."
             )
-        if args.no_binary:
+        if args.no_black_binary and "black" in package_name:
             pip_args.append(f"--no-binary={package_name}")
 
     dry_run = args.dry_run == "1"