From 6935b723f7621593f06061d2be0f1d0a44ec8e37 Mon Sep 17 00:00:00 2001 From: saymrwulf Date: Mon, 10 Feb 2025 17:26:00 +0100 Subject: [PATCH] Created using Colab --- RL_countdown_r1zero_01.ipynb | 3699 ++++++++++++++++++++++++++++++++++ 1 file changed, 3699 insertions(+) create mode 100644 RL_countdown_r1zero_01.ipynb diff --git a/RL_countdown_r1zero_01.ipynb b/RL_countdown_r1zero_01.ipynb new file mode 100644 index 0000000..fd08e37 --- /dev/null +++ b/RL_countdown_r1zero_01.ipynb @@ -0,0 +1,3699 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [], + "machine_shape": "hm", + "gpuType": "T4", + "collapsed_sections": [ + "h7Bggr7moWnN", + "rXHkrMKfYzBw", + "FS9J3e3wZBzt", + "aLB0VEyda4Wr", + "KXwc1QgDeVSL", + "IMVU7nK3cmyp", + "krUPUCDJoupx", + "DalOsGS6cT-X", + "enstPlUcgV-n" + ], + "authorship_tag": "ABX9TyOYzri8xbFh0VLZMOrp2hgC", + "include_colab_link": true + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + }, + "accelerator": "GPU" + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "view-in-github", + "colab_type": "text" + }, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "markdown", + "source": [ + "# Prep" + ], + "metadata": { + "id": "_jQr2rS1YP-I" + } + }, + { + "cell_type": "markdown", + "source": [ + "###TinyZero Github" + ], + "metadata": { + "id": "h7Bggr7moWnN" + } + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "RfrqCQmtFPeX", + "outputId": "b7761d4d-266d-4337-e1f5-c9af5057a4bc" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "(True, 'Tesla T4')" + ] + }, + "metadata": {}, + "execution_count": 1 + } + ], + "source": [ + "import torch\n", + "torch.cuda.is_available(), torch.cuda.get_device_name(0)" + ] + }, + { + "cell_type": "code", + "source": [ + "!git clone https://github.com/Jiayi-Pan/TinyZero.git\n", + "%cd TinyZero" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "C5e2k4naG0t5", + "outputId": "c3c5087f-1f41-4923-a6e5-be38db7a71fa", + "collapsed": true + }, + "execution_count": 2, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Cloning into 'TinyZero'...\n", + "remote: Enumerating objects: 1659, done.\u001b[K\n", + "remote: Counting objects: 100% (485/485), done.\u001b[K\n", + "remote: Compressing objects: 100% (97/97), done.\u001b[K\n", + "remote: Total 1659 (delta 422), reused 388 (delta 388), pack-reused 1174 (from 1)\u001b[K\n", + "Receiving objects: 100% (1659/1659), 2.33 MiB | 27.76 MiB/s, done.\n", + "Resolving deltas: 100% (1017/1017), done.\n", + "/content/TinyZero\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "!pip install verl IPython matplotlib\n", + "!pip install -r requirements.txt" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 0 + }, + "collapsed": true, + "id": "h943eaG2G9r0", + "outputId": "07324d1e-5537-4bcc-92b9-ca65211bd02e" + }, + "execution_count": 3, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Collecting verl\n", + " Downloading verl-0.1-py2.py3-none-any.whl.metadata (9.9 kB)\n", + "Requirement already satisfied: IPython in /usr/local/lib/python3.11/dist-packages (7.34.0)\n", + "Requirement already satisfied: matplotlib in /usr/local/lib/python3.11/dist-packages (3.10.0)\n", + "Collecting torch==2.4.0 (from verl)\n", + " Downloading torch-2.4.0-cp311-cp311-manylinux1_x86_64.whl.metadata (26 kB)\n", + "Collecting tensordict (from verl)\n", + " Downloading tensordict-0.7.0-cp311-cp311-manylinux1_x86_64.whl.metadata (9.1 kB)\n", + "Requirement already satisfied: transformers in /usr/local/lib/python3.11/dist-packages (from verl) (4.48.2)\n", + "Collecting codetiming (from verl)\n", + " Downloading codetiming-1.4.0-py3-none-any.whl.metadata (7.7 kB)\n", + "Collecting pybind11 (from verl)\n", + " Downloading pybind11-2.13.6-py3-none-any.whl.metadata (9.5 kB)\n", + "Collecting hydra-core (from verl)\n", + " Downloading hydra_core-1.3.2-py3-none-any.whl.metadata (5.5 kB)\n", + "Requirement already satisfied: numpy in /usr/local/lib/python3.11/dist-packages (from verl) (1.26.4)\n", + "Collecting yapf (from verl)\n", + " Downloading yapf-0.43.0-py3-none-any.whl.metadata (46 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.8/46.8 kB\u001b[0m \u001b[31m4.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting dill (from verl)\n", + " Downloading dill-0.3.9-py3-none-any.whl.metadata (10 kB)\n", + "Requirement already satisfied: accelerate in /usr/local/lib/python3.11/dist-packages (from verl) (1.3.0)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.11/dist-packages (from torch==2.4.0->verl) (3.17.0)\n", + "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.11/dist-packages (from torch==2.4.0->verl) (4.12.2)\n", + "Requirement already satisfied: sympy in /usr/local/lib/python3.11/dist-packages (from torch==2.4.0->verl) (1.13.1)\n", + "Requirement already satisfied: networkx in /usr/local/lib/python3.11/dist-packages (from torch==2.4.0->verl) (3.4.2)\n", + "Requirement already satisfied: jinja2 in /usr/local/lib/python3.11/dist-packages (from torch==2.4.0->verl) (3.1.5)\n", + "Requirement already satisfied: fsspec in /usr/local/lib/python3.11/dist-packages (from torch==2.4.0->verl) (2024.10.0)\n", + "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch==2.4.0->verl)\n", + " Downloading nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n", + "Collecting nvidia-cuda-runtime-cu12==12.1.105 (from torch==2.4.0->verl)\n", + " Downloading nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n", + "Collecting nvidia-cuda-cupti-cu12==12.1.105 (from torch==2.4.0->verl)\n", + " Downloading nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n", + "Collecting nvidia-cudnn-cu12==9.1.0.70 (from torch==2.4.0->verl)\n", + " Downloading nvidia_cudnn_cu12-9.1.0.70-py3-none-manylinux2014_x86_64.whl.metadata (1.6 kB)\n", + "Collecting nvidia-cublas-cu12==12.1.3.1 (from torch==2.4.0->verl)\n", + " Downloading nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n", + "Collecting nvidia-cufft-cu12==11.0.2.54 (from torch==2.4.0->verl)\n", + " Downloading nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n", + "Collecting nvidia-curand-cu12==10.3.2.106 (from torch==2.4.0->verl)\n", + " Downloading nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n", + "Collecting nvidia-cusolver-cu12==11.4.5.107 (from torch==2.4.0->verl)\n", + " Downloading nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n", + "Collecting nvidia-cusparse-cu12==12.1.0.106 (from torch==2.4.0->verl)\n", + " Downloading nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n", + "Collecting nvidia-nccl-cu12==2.20.5 (from torch==2.4.0->verl)\n", + " Downloading nvidia_nccl_cu12-2.20.5-py3-none-manylinux2014_x86_64.whl.metadata (1.8 kB)\n", + "Collecting nvidia-nvtx-cu12==12.1.105 (from torch==2.4.0->verl)\n", + " Downloading nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.7 kB)\n", + "Collecting triton==3.0.0 (from torch==2.4.0->verl)\n", + " Downloading triton-3.0.0-1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl.metadata (1.3 kB)\n", + "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/lib/python3.11/dist-packages (from nvidia-cusolver-cu12==11.4.5.107->torch==2.4.0->verl) (12.5.82)\n", + "Requirement already satisfied: setuptools>=18.5 in /usr/local/lib/python3.11/dist-packages (from IPython) (75.1.0)\n", + "Collecting jedi>=0.16 (from IPython)\n", + " Downloading jedi-0.19.2-py2.py3-none-any.whl.metadata (22 kB)\n", + "Requirement already satisfied: decorator in /usr/local/lib/python3.11/dist-packages (from IPython) (4.4.2)\n", + "Requirement already satisfied: pickleshare in /usr/local/lib/python3.11/dist-packages (from IPython) (0.7.5)\n", + "Requirement already satisfied: traitlets>=4.2 in /usr/local/lib/python3.11/dist-packages (from IPython) (5.7.1)\n", + "Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in /usr/local/lib/python3.11/dist-packages (from IPython) (3.0.50)\n", + "Requirement already satisfied: pygments in /usr/local/lib/python3.11/dist-packages (from IPython) (2.18.0)\n", + "Requirement already satisfied: backcall in /usr/local/lib/python3.11/dist-packages (from IPython) (0.2.0)\n", + "Requirement already satisfied: matplotlib-inline in /usr/local/lib/python3.11/dist-packages (from IPython) (0.1.7)\n", + "Requirement already satisfied: pexpect>4.3 in /usr/local/lib/python3.11/dist-packages (from IPython) (4.9.0)\n", + "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (1.3.1)\n", + "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (0.12.1)\n", + "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (4.55.8)\n", + "Requirement already satisfied: kiwisolver>=1.3.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (1.4.8)\n", + "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (24.2)\n", + "Requirement already satisfied: pillow>=8 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (11.1.0)\n", + "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (3.2.1)\n", + "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.11/dist-packages (from matplotlib) (2.8.2)\n", + "Requirement already satisfied: parso<0.9.0,>=0.8.4 in /usr/local/lib/python3.11/dist-packages (from jedi>=0.16->IPython) (0.8.4)\n", + "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.11/dist-packages (from pexpect>4.3->IPython) (0.7.0)\n", + "Requirement already satisfied: wcwidth in /usr/local/lib/python3.11/dist-packages (from prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0->IPython) (0.2.13)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.11/dist-packages (from python-dateutil>=2.7->matplotlib) (1.17.0)\n", + "Requirement already satisfied: psutil in /usr/local/lib/python3.11/dist-packages (from accelerate->verl) (5.9.5)\n", + "Requirement already satisfied: pyyaml in /usr/local/lib/python3.11/dist-packages (from accelerate->verl) (6.0.2)\n", + "Requirement already satisfied: huggingface-hub>=0.21.0 in /usr/local/lib/python3.11/dist-packages (from accelerate->verl) (0.28.1)\n", + "Requirement already satisfied: safetensors>=0.4.3 in /usr/local/lib/python3.11/dist-packages (from accelerate->verl) (0.5.2)\n", + "Collecting omegaconf<2.4,>=2.2 (from hydra-core->verl)\n", + " Downloading omegaconf-2.3.0-py3-none-any.whl.metadata (3.9 kB)\n", + "Collecting antlr4-python3-runtime==4.9.* (from hydra-core->verl)\n", + " Downloading antlr4-python3-runtime-4.9.3.tar.gz (117 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m117.0/117.0 kB\u001b[0m \u001b[31m10.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + "INFO: pip is looking at multiple versions of tensordict to determine which version is compatible with other requirements. This could take a while.\n", + "Collecting tensordict (from verl)\n", + " Downloading tensordict-0.6.2-cp311-cp311-manylinux1_x86_64.whl.metadata (8.9 kB)\n", + " Downloading tensordict-0.6.1-cp311-cp311-manylinux1_x86_64.whl.metadata (8.9 kB)\n", + " Downloading tensordict-0.6.0-cp311-cp311-manylinux1_x86_64.whl.metadata (8.9 kB)\n", + " Downloading tensordict-0.5.0-cp311-cp311-manylinux1_x86_64.whl.metadata (22 kB)\n", + "Requirement already satisfied: cloudpickle in /usr/local/lib/python3.11/dist-packages (from tensordict->verl) (3.1.1)\n", + "Requirement already satisfied: orjson in /usr/local/lib/python3.11/dist-packages (from tensordict->verl) (3.10.15)\n", + "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.11/dist-packages (from transformers->verl) (2024.11.6)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.11/dist-packages (from transformers->verl) (2.32.3)\n", + "Requirement already satisfied: tokenizers<0.22,>=0.21 in /usr/local/lib/python3.11/dist-packages (from transformers->verl) (0.21.0)\n", + "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.11/dist-packages (from transformers->verl) (4.67.1)\n", + "Requirement already satisfied: platformdirs>=3.5.1 in /usr/local/lib/python3.11/dist-packages (from yapf->verl) (4.3.6)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.11/dist-packages (from jinja2->torch==2.4.0->verl) (3.0.2)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/dist-packages (from requests->transformers->verl) (3.4.1)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/dist-packages (from requests->transformers->verl) (3.10)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/dist-packages (from requests->transformers->verl) (2.3.0)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/dist-packages (from requests->transformers->verl) (2025.1.31)\n", + "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.11/dist-packages (from sympy->torch==2.4.0->verl) (1.3.0)\n", + "Downloading verl-0.1-py2.py3-none-any.whl (371 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m371.5/371.5 kB\u001b[0m \u001b[31m30.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading torch-2.4.0-cp311-cp311-manylinux1_x86_64.whl (797.3 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m797.3/797.3 MB\u001b[0m \u001b[31m2.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m410.6/410.6 MB\u001b[0m \u001b[31m2.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.1/14.1 MB\u001b[0m \u001b[31m114.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m23.7/23.7 MB\u001b[0m \u001b[31m97.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m823.6/823.6 kB\u001b[0m \u001b[31m57.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading nvidia_cudnn_cu12-9.1.0.70-py3-none-manylinux2014_x86_64.whl (664.8 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m664.8/664.8 MB\u001b[0m \u001b[31m2.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.6/121.6 MB\u001b[0m \u001b[31m18.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.5/56.5 MB\u001b[0m \u001b[31m40.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m124.2/124.2 MB\u001b[0m \u001b[31m7.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m196.0/196.0 MB\u001b[0m \u001b[31m4.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading nvidia_nccl_cu12-2.20.5-py3-none-manylinux2014_x86_64.whl (176.2 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m176.2/176.2 MB\u001b[0m \u001b[31m8.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m99.1/99.1 kB\u001b[0m \u001b[31m9.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading triton-3.0.0-1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (209.4 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m209.4/209.4 MB\u001b[0m \u001b[31m5.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading jedi-0.19.2-py2.py3-none-any.whl (1.6 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m67.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading codetiming-1.4.0-py3-none-any.whl (7.2 kB)\n", + "Downloading dill-0.3.9-py3-none-any.whl (119 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m119.4/119.4 kB\u001b[0m \u001b[31m10.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading hydra_core-1.3.2-py3-none-any.whl (154 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m154.5/154.5 kB\u001b[0m \u001b[31m12.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading pybind11-2.13.6-py3-none-any.whl (243 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m243.3/243.3 kB\u001b[0m \u001b[31m20.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading tensordict-0.5.0-cp311-cp311-manylinux1_x86_64.whl (337 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m337.1/337.1 kB\u001b[0m \u001b[31m29.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading yapf-0.43.0-py3-none-any.whl (256 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m256.2/256.2 kB\u001b[0m \u001b[31m23.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading omegaconf-2.3.0-py3-none-any.whl (79 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m79.5/79.5 kB\u001b[0m \u001b[31m8.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hBuilding wheels for collected packages: antlr4-python3-runtime\n", + " Building wheel for antlr4-python3-runtime (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for antlr4-python3-runtime: filename=antlr4_python3_runtime-4.9.3-py3-none-any.whl size=144555 sha256=b539d656af1f6a3ef2a36002a19d757548a804dc58314e1458ca2213c5c7ee59\n", + " Stored in directory: /root/.cache/pip/wheels/1a/97/32/461f837398029ad76911109f07047fde1d7b661a147c7c56d1\n", + "Successfully built antlr4-python3-runtime\n", + "Installing collected packages: antlr4-python3-runtime, yapf, triton, pybind11, omegaconf, nvidia-nvtx-cu12, nvidia-nccl-cu12, nvidia-cusparse-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, jedi, dill, codetiming, nvidia-cusolver-cu12, nvidia-cudnn-cu12, hydra-core, torch, tensordict, verl\n", + " Attempting uninstall: triton\n", + " Found existing installation: triton 3.1.0\n", + " Uninstalling triton-3.1.0:\n", + " Successfully uninstalled triton-3.1.0\n", + " Attempting uninstall: nvidia-nvtx-cu12\n", + " Found existing installation: nvidia-nvtx-cu12 12.4.127\n", + " Uninstalling nvidia-nvtx-cu12-12.4.127:\n", + " Successfully uninstalled nvidia-nvtx-cu12-12.4.127\n", + " Attempting uninstall: nvidia-nccl-cu12\n", + " Found existing installation: nvidia-nccl-cu12 2.21.5\n", + " Uninstalling nvidia-nccl-cu12-2.21.5:\n", + " Successfully uninstalled nvidia-nccl-cu12-2.21.5\n", + " Attempting uninstall: nvidia-cusparse-cu12\n", + " Found existing installation: nvidia-cusparse-cu12 12.5.1.3\n", + " Uninstalling nvidia-cusparse-cu12-12.5.1.3:\n", + " Successfully uninstalled nvidia-cusparse-cu12-12.5.1.3\n", + " Attempting uninstall: nvidia-curand-cu12\n", + " Found existing installation: nvidia-curand-cu12 10.3.6.82\n", + " Uninstalling nvidia-curand-cu12-10.3.6.82:\n", + " Successfully uninstalled nvidia-curand-cu12-10.3.6.82\n", + " Attempting uninstall: nvidia-cufft-cu12\n", + " Found existing installation: nvidia-cufft-cu12 11.2.3.61\n", + " Uninstalling nvidia-cufft-cu12-11.2.3.61:\n", + " Successfully uninstalled nvidia-cufft-cu12-11.2.3.61\n", + " Attempting uninstall: nvidia-cuda-runtime-cu12\n", + " Found existing installation: nvidia-cuda-runtime-cu12 12.5.82\n", + " Uninstalling nvidia-cuda-runtime-cu12-12.5.82:\n", + " Successfully uninstalled nvidia-cuda-runtime-cu12-12.5.82\n", + " Attempting uninstall: nvidia-cuda-nvrtc-cu12\n", + " Found existing installation: nvidia-cuda-nvrtc-cu12 12.5.82\n", + " Uninstalling nvidia-cuda-nvrtc-cu12-12.5.82:\n", + " Successfully uninstalled nvidia-cuda-nvrtc-cu12-12.5.82\n", + " Attempting uninstall: nvidia-cuda-cupti-cu12\n", + " Found existing installation: nvidia-cuda-cupti-cu12 12.5.82\n", + " Uninstalling nvidia-cuda-cupti-cu12-12.5.82:\n", + " Successfully uninstalled nvidia-cuda-cupti-cu12-12.5.82\n", + " Attempting uninstall: nvidia-cublas-cu12\n", + " Found existing installation: nvidia-cublas-cu12 12.5.3.2\n", + " Uninstalling nvidia-cublas-cu12-12.5.3.2:\n", + " Successfully uninstalled nvidia-cublas-cu12-12.5.3.2\n", + " Attempting uninstall: nvidia-cusolver-cu12\n", + " Found existing installation: nvidia-cusolver-cu12 11.6.3.83\n", + " Uninstalling nvidia-cusolver-cu12-11.6.3.83:\n", + " Successfully uninstalled nvidia-cusolver-cu12-11.6.3.83\n", + " Attempting uninstall: nvidia-cudnn-cu12\n", + " Found existing installation: nvidia-cudnn-cu12 9.3.0.75\n", + " Uninstalling nvidia-cudnn-cu12-9.3.0.75:\n", + " Successfully uninstalled nvidia-cudnn-cu12-9.3.0.75\n", + " Attempting uninstall: torch\n", + " Found existing installation: torch 2.5.1+cu124\n", + " Uninstalling torch-2.5.1+cu124:\n", + " Successfully uninstalled torch-2.5.1+cu124\n", + "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n", + "torchvision 0.20.1+cu124 requires torch==2.5.1, but you have torch 2.4.0 which is incompatible.\n", + "torchaudio 2.5.1+cu124 requires torch==2.5.1, but you have torch 2.4.0 which is incompatible.\u001b[0m\u001b[31m\n", + "\u001b[0mSuccessfully installed antlr4-python3-runtime-4.9.3 codetiming-1.4.0 dill-0.3.9 hydra-core-1.3.2 jedi-0.19.2 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-9.1.0.70 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.20.5 nvidia-nvtx-cu12-12.1.105 omegaconf-2.3.0 pybind11-2.13.6 tensordict-0.5.0 torch-2.4.0 triton-3.0.0 verl-0.1 yapf-0.43.0\n" + ] + }, + { + "output_type": "display_data", + "data": { + "application/vnd.colab-display-data+json": { + "pip_warning": { + "packages": [ + "pydevd_plugins", + "torch", + "torchgen" + ] + }, + "id": "112b3cd9c1c14af6b16e0ad077b6ab55" + } + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Requirement already satisfied: accelerate in /usr/local/lib/python3.11/dist-packages (from -r requirements.txt (line 1)) (1.3.0)\n", + "Requirement already satisfied: codetiming in /usr/local/lib/python3.11/dist-packages (from -r requirements.txt (line 2)) (1.4.0)\n", + "Collecting datasets (from -r requirements.txt (line 3))\n", + " Downloading datasets-3.2.0-py3-none-any.whl.metadata (20 kB)\n", + "Requirement already satisfied: dill in /usr/local/lib/python3.11/dist-packages (from -r requirements.txt (line 4)) (0.3.9)\n", + "Collecting flash-attn (from -r requirements.txt (line 5))\n", + " Downloading flash_attn-2.7.4.post1.tar.gz (6.0 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.0/6.0 MB\u001b[0m \u001b[31m72.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + "Requirement already satisfied: hydra-core in /usr/local/lib/python3.11/dist-packages (from -r requirements.txt (line 6)) (1.3.2)\n", + "Requirement already satisfied: numpy in /usr/local/lib/python3.11/dist-packages (from -r requirements.txt (line 7)) (1.26.4)\n", + "Requirement already satisfied: pandas in /usr/local/lib/python3.11/dist-packages (from -r requirements.txt (line 8)) (2.2.2)\n", + "Requirement already satisfied: pybind11 in /usr/local/lib/python3.11/dist-packages (from -r requirements.txt (line 9)) (2.13.6)\n", + "Collecting ray (from -r requirements.txt (line 10))\n", + " Downloading ray-2.42.0-cp311-cp311-manylinux2014_x86_64.whl.metadata (18 kB)\n", + "Requirement already satisfied: tensordict<0.6 in /usr/local/lib/python3.11/dist-packages (from -r requirements.txt (line 11)) (0.5.0)\n", + "Collecting transformers<4.48 (from -r requirements.txt (line 12))\n", + " Downloading transformers-4.47.1-py3-none-any.whl.metadata (44 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m44.1/44.1 kB\u001b[0m \u001b[31m4.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting vllm<=0.6.3 (from -r requirements.txt (line 13))\n", + " Downloading vllm-0.6.3-cp38-abi3-manylinux1_x86_64.whl.metadata (10 kB)\n", + "Requirement already satisfied: wandb in /usr/local/lib/python3.11/dist-packages (from -r requirements.txt (line 14)) (0.19.6)\n", + "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.11/dist-packages (from accelerate->-r requirements.txt (line 1)) (24.2)\n", + "Requirement already satisfied: psutil in /usr/local/lib/python3.11/dist-packages (from accelerate->-r requirements.txt (line 1)) (5.9.5)\n", + "Requirement already satisfied: pyyaml in /usr/local/lib/python3.11/dist-packages (from accelerate->-r requirements.txt (line 1)) (6.0.2)\n", + "Requirement already satisfied: torch>=2.0.0 in /usr/local/lib/python3.11/dist-packages (from accelerate->-r requirements.txt (line 1)) (2.4.0)\n", + "Requirement already satisfied: huggingface-hub>=0.21.0 in /usr/local/lib/python3.11/dist-packages (from accelerate->-r requirements.txt (line 1)) (0.28.1)\n", + "Requirement already satisfied: safetensors>=0.4.3 in /usr/local/lib/python3.11/dist-packages (from accelerate->-r requirements.txt (line 1)) (0.5.2)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.11/dist-packages (from datasets->-r requirements.txt (line 3)) (3.17.0)\n", + "Requirement already satisfied: pyarrow>=15.0.0 in /usr/local/lib/python3.11/dist-packages (from datasets->-r requirements.txt (line 3)) (17.0.0)\n", + "Collecting dill (from -r requirements.txt (line 4))\n", + " Downloading dill-0.3.8-py3-none-any.whl.metadata (10 kB)\n", + "Requirement already satisfied: requests>=2.32.2 in /usr/local/lib/python3.11/dist-packages (from datasets->-r requirements.txt (line 3)) (2.32.3)\n", + "Requirement already satisfied: tqdm>=4.66.3 in /usr/local/lib/python3.11/dist-packages (from datasets->-r requirements.txt (line 3)) (4.67.1)\n", + "Collecting xxhash (from datasets->-r requirements.txt (line 3))\n", + " Downloading xxhash-3.5.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (12 kB)\n", + "Collecting multiprocess<0.70.17 (from datasets->-r requirements.txt (line 3))\n", + " Downloading multiprocess-0.70.16-py311-none-any.whl.metadata (7.2 kB)\n", + "Collecting fsspec<=2024.9.0,>=2023.1.0 (from fsspec[http]<=2024.9.0,>=2023.1.0->datasets->-r requirements.txt (line 3))\n", + " Downloading fsspec-2024.9.0-py3-none-any.whl.metadata (11 kB)\n", + "Requirement already satisfied: aiohttp in /usr/local/lib/python3.11/dist-packages (from datasets->-r requirements.txt (line 3)) (3.11.11)\n", + "Requirement already satisfied: einops in /usr/local/lib/python3.11/dist-packages (from flash-attn->-r requirements.txt (line 5)) (0.8.0)\n", + "Requirement already satisfied: omegaconf<2.4,>=2.2 in /usr/local/lib/python3.11/dist-packages (from hydra-core->-r requirements.txt (line 6)) (2.3.0)\n", + "Requirement already satisfied: antlr4-python3-runtime==4.9.* in /usr/local/lib/python3.11/dist-packages (from hydra-core->-r requirements.txt (line 6)) (4.9.3)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/dist-packages (from pandas->-r requirements.txt (line 8)) (2.8.2)\n", + "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/dist-packages (from pandas->-r requirements.txt (line 8)) (2025.1)\n", + "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/dist-packages (from pandas->-r requirements.txt (line 8)) (2025.1)\n", + "Requirement already satisfied: click>=7.0 in /usr/local/lib/python3.11/dist-packages (from ray->-r requirements.txt (line 10)) (8.1.8)\n", + "Requirement already satisfied: jsonschema in /usr/local/lib/python3.11/dist-packages (from ray->-r requirements.txt (line 10)) (4.23.0)\n", + "Requirement already satisfied: msgpack<2.0.0,>=1.0.0 in /usr/local/lib/python3.11/dist-packages (from ray->-r requirements.txt (line 10)) (1.1.0)\n", + "Requirement already satisfied: protobuf!=3.19.5,>=3.15.3 in /usr/local/lib/python3.11/dist-packages (from ray->-r requirements.txt (line 10)) (4.25.6)\n", + "Requirement already satisfied: aiosignal in /usr/local/lib/python3.11/dist-packages (from ray->-r requirements.txt (line 10)) (1.3.2)\n", + "Requirement already satisfied: frozenlist in /usr/local/lib/python3.11/dist-packages (from ray->-r requirements.txt (line 10)) (1.5.0)\n", + "Requirement already satisfied: cloudpickle in /usr/local/lib/python3.11/dist-packages (from tensordict<0.6->-r requirements.txt (line 11)) (3.1.1)\n", + "Requirement already satisfied: orjson in /usr/local/lib/python3.11/dist-packages (from tensordict<0.6->-r requirements.txt (line 11)) (3.10.15)\n", + "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.11/dist-packages (from transformers<4.48->-r requirements.txt (line 12)) (2024.11.6)\n", + "Requirement already satisfied: tokenizers<0.22,>=0.21 in /usr/local/lib/python3.11/dist-packages (from transformers<4.48->-r requirements.txt (line 12)) (0.21.0)\n", + "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.11/dist-packages (from vllm<=0.6.3->-r requirements.txt (line 13)) (0.2.0)\n", + "Requirement already satisfied: py-cpuinfo in /usr/local/lib/python3.11/dist-packages (from vllm<=0.6.3->-r requirements.txt (line 13)) (9.0.0)\n", + "Requirement already satisfied: openai>=1.40.0 in /usr/local/lib/python3.11/dist-packages (from vllm<=0.6.3->-r requirements.txt (line 13)) (1.61.1)\n", + "Collecting uvicorn[standard] (from vllm<=0.6.3->-r requirements.txt (line 13))\n", + " Downloading uvicorn-0.34.0-py3-none-any.whl.metadata (6.5 kB)\n", + "Requirement already satisfied: pydantic>=2.9 in /usr/local/lib/python3.11/dist-packages (from vllm<=0.6.3->-r requirements.txt (line 13)) (2.10.6)\n", + "Requirement already satisfied: pillow in /usr/local/lib/python3.11/dist-packages (from vllm<=0.6.3->-r requirements.txt (line 13)) (11.1.0)\n", + "Requirement already satisfied: prometheus-client>=0.18.0 in /usr/local/lib/python3.11/dist-packages (from vllm<=0.6.3->-r requirements.txt (line 13)) (0.21.1)\n", + "Collecting prometheus-fastapi-instrumentator>=7.0.0 (from vllm<=0.6.3->-r requirements.txt (line 13))\n", + " Downloading prometheus_fastapi_instrumentator-7.0.2-py3-none-any.whl.metadata (13 kB)\n", + "Collecting tiktoken>=0.6.0 (from vllm<=0.6.3->-r requirements.txt (line 13))\n", + " Downloading tiktoken-0.8.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.6 kB)\n", + "Collecting lm-format-enforcer==0.10.6 (from vllm<=0.6.3->-r requirements.txt (line 13))\n", + " Downloading lm_format_enforcer-0.10.6-py3-none-any.whl.metadata (16 kB)\n", + "Collecting outlines<0.1,>=0.0.43 (from vllm<=0.6.3->-r requirements.txt (line 13))\n", + " Downloading outlines-0.0.46-py3-none-any.whl.metadata (15 kB)\n", + "Requirement already satisfied: typing-extensions>=4.10 in /usr/local/lib/python3.11/dist-packages (from vllm<=0.6.3->-r requirements.txt (line 13)) (4.12.2)\n", + "Collecting partial-json-parser (from vllm<=0.6.3->-r requirements.txt (line 13))\n", + " Downloading partial_json_parser-0.2.1.1.post5-py3-none-any.whl.metadata (6.1 kB)\n", + "Requirement already satisfied: pyzmq in /usr/local/lib/python3.11/dist-packages (from vllm<=0.6.3->-r requirements.txt (line 13)) (24.0.1)\n", + "Collecting msgspec (from vllm<=0.6.3->-r requirements.txt (line 13))\n", + " Downloading msgspec-0.19.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.9 kB)\n", + "Collecting gguf==0.10.0 (from vllm<=0.6.3->-r requirements.txt (line 13))\n", + " Downloading gguf-0.10.0-py3-none-any.whl.metadata (3.5 kB)\n", + "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.11/dist-packages (from vllm<=0.6.3->-r requirements.txt (line 13)) (8.6.1)\n", + "Collecting mistral-common>=1.4.4 (from mistral-common[opencv]>=1.4.4->vllm<=0.6.3->-r requirements.txt (line 13))\n", + " Downloading mistral_common-1.5.3-py3-none-any.whl.metadata (4.5 kB)\n", + "Collecting nvidia-ml-py (from vllm<=0.6.3->-r requirements.txt (line 13))\n", + " Downloading nvidia_ml_py-12.570.86-py3-none-any.whl.metadata (8.7 kB)\n", + "Collecting torchvision==0.19 (from vllm<=0.6.3->-r requirements.txt (line 13))\n", + " Downloading torchvision-0.19.0-cp311-cp311-manylinux1_x86_64.whl.metadata (6.0 kB)\n", + "Collecting xformers==0.0.27.post2 (from vllm<=0.6.3->-r requirements.txt (line 13))\n", + " Downloading xformers-0.0.27.post2-cp311-cp311-manylinux2014_x86_64.whl.metadata (1.0 kB)\n", + "Collecting fastapi!=0.113.*,!=0.114.0,>=0.107.0 (from vllm<=0.6.3->-r requirements.txt (line 13))\n", + " Downloading fastapi-0.115.8-py3-none-any.whl.metadata (27 kB)\n", + "Collecting interegular>=0.3.2 (from lm-format-enforcer==0.10.6->vllm<=0.6.3->-r requirements.txt (line 13))\n", + " Downloading interegular-0.3.3-py37-none-any.whl.metadata (3.0 kB)\n", + "Requirement already satisfied: sympy in /usr/local/lib/python3.11/dist-packages (from torch>=2.0.0->accelerate->-r requirements.txt (line 1)) (1.13.1)\n", + "Requirement already satisfied: networkx in /usr/local/lib/python3.11/dist-packages (from torch>=2.0.0->accelerate->-r requirements.txt (line 1)) (3.4.2)\n", + "Requirement already satisfied: jinja2 in /usr/local/lib/python3.11/dist-packages (from torch>=2.0.0->accelerate->-r requirements.txt (line 1)) (3.1.5)\n", + "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/lib/python3.11/dist-packages (from torch>=2.0.0->accelerate->-r requirements.txt (line 1)) (12.1.105)\n", + "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/lib/python3.11/dist-packages (from torch>=2.0.0->accelerate->-r requirements.txt (line 1)) (12.1.105)\n", + "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/lib/python3.11/dist-packages (from torch>=2.0.0->accelerate->-r requirements.txt (line 1)) (12.1.105)\n", + "Requirement already satisfied: nvidia-cudnn-cu12==9.1.0.70 in /usr/local/lib/python3.11/dist-packages (from torch>=2.0.0->accelerate->-r requirements.txt (line 1)) (9.1.0.70)\n", + "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/lib/python3.11/dist-packages (from torch>=2.0.0->accelerate->-r requirements.txt (line 1)) (12.1.3.1)\n", + "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/lib/python3.11/dist-packages (from torch>=2.0.0->accelerate->-r requirements.txt (line 1)) (11.0.2.54)\n", + "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/lib/python3.11/dist-packages (from torch>=2.0.0->accelerate->-r requirements.txt (line 1)) (10.3.2.106)\n", + "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/lib/python3.11/dist-packages (from torch>=2.0.0->accelerate->-r requirements.txt (line 1)) (11.4.5.107)\n", + "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/lib/python3.11/dist-packages (from torch>=2.0.0->accelerate->-r requirements.txt (line 1)) (12.1.0.106)\n", + "Requirement already satisfied: nvidia-nccl-cu12==2.20.5 in /usr/local/lib/python3.11/dist-packages (from torch>=2.0.0->accelerate->-r requirements.txt (line 1)) (2.20.5)\n", + "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/lib/python3.11/dist-packages (from torch>=2.0.0->accelerate->-r requirements.txt (line 1)) (12.1.105)\n", + "Requirement already satisfied: triton==3.0.0 in /usr/local/lib/python3.11/dist-packages (from torch>=2.0.0->accelerate->-r requirements.txt (line 1)) (3.0.0)\n", + "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/lib/python3.11/dist-packages (from nvidia-cusolver-cu12==11.4.5.107->torch>=2.0.0->accelerate->-r requirements.txt (line 1)) (12.5.82)\n", + "Requirement already satisfied: docker-pycreds>=0.4.0 in /usr/local/lib/python3.11/dist-packages (from wandb->-r requirements.txt (line 14)) (0.4.0)\n", + "Requirement already satisfied: gitpython!=3.1.29,>=1.0.0 in /usr/local/lib/python3.11/dist-packages (from wandb->-r requirements.txt (line 14)) (3.1.44)\n", + "Requirement already satisfied: platformdirs in /usr/local/lib/python3.11/dist-packages (from wandb->-r requirements.txt (line 14)) (4.3.6)\n", + "Requirement already satisfied: sentry-sdk>=2.0.0 in /usr/local/lib/python3.11/dist-packages (from wandb->-r requirements.txt (line 14)) (2.20.0)\n", + "Requirement already satisfied: setproctitle in /usr/local/lib/python3.11/dist-packages (from wandb->-r requirements.txt (line 14)) (1.3.4)\n", + "Requirement already satisfied: setuptools in /usr/local/lib/python3.11/dist-packages (from wandb->-r requirements.txt (line 14)) (75.1.0)\n", + "Requirement already satisfied: six>=1.4.0 in /usr/local/lib/python3.11/dist-packages (from docker-pycreds>=0.4.0->wandb->-r requirements.txt (line 14)) (1.17.0)\n", + "Collecting starlette<0.46.0,>=0.40.0 (from fastapi!=0.113.*,!=0.114.0,>=0.107.0->vllm<=0.6.3->-r requirements.txt (line 13))\n", + " Downloading starlette-0.45.3-py3-none-any.whl.metadata (6.3 kB)\n", + "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets->-r requirements.txt (line 3)) (2.4.4)\n", + "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets->-r requirements.txt (line 3)) (25.1.0)\n", + "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets->-r requirements.txt (line 3)) (6.1.0)\n", + "Requirement already satisfied: propcache>=0.2.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets->-r requirements.txt (line 3)) (0.2.1)\n", + "Requirement already satisfied: yarl<2.0,>=1.17.0 in /usr/local/lib/python3.11/dist-packages (from aiohttp->datasets->-r requirements.txt (line 3)) (1.18.3)\n", + "Requirement already satisfied: gitdb<5,>=4.0.1 in /usr/local/lib/python3.11/dist-packages (from gitpython!=3.1.29,>=1.0.0->wandb->-r requirements.txt (line 14)) (4.0.12)\n", + "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /usr/local/lib/python3.11/dist-packages (from jsonschema->ray->-r requirements.txt (line 10)) (2024.10.1)\n", + "Requirement already satisfied: referencing>=0.28.4 in /usr/local/lib/python3.11/dist-packages (from jsonschema->ray->-r requirements.txt (line 10)) (0.36.2)\n", + "Requirement already satisfied: rpds-py>=0.7.1 in /usr/local/lib/python3.11/dist-packages (from jsonschema->ray->-r requirements.txt (line 10)) (0.22.3)\n", + "Requirement already satisfied: opencv-python-headless>=4.0.0 in /usr/local/lib/python3.11/dist-packages (from mistral-common[opencv]>=1.4.4->vllm<=0.6.3->-r requirements.txt (line 13)) (4.11.0.86)\n", + "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.11/dist-packages (from openai>=1.40.0->vllm<=0.6.3->-r requirements.txt (line 13)) (3.7.1)\n", + "Requirement already satisfied: distro<2,>=1.7.0 in /usr/local/lib/python3.11/dist-packages (from openai>=1.40.0->vllm<=0.6.3->-r requirements.txt (line 13)) (1.9.0)\n", + "Requirement already satisfied: httpx<1,>=0.23.0 in /usr/local/lib/python3.11/dist-packages (from openai>=1.40.0->vllm<=0.6.3->-r requirements.txt (line 13)) (0.28.1)\n", + "Requirement already satisfied: jiter<1,>=0.4.0 in /usr/local/lib/python3.11/dist-packages (from openai>=1.40.0->vllm<=0.6.3->-r requirements.txt (line 13)) (0.8.2)\n", + "Requirement already satisfied: sniffio in /usr/local/lib/python3.11/dist-packages (from openai>=1.40.0->vllm<=0.6.3->-r requirements.txt (line 13)) (1.3.1)\n", + "Collecting lark (from outlines<0.1,>=0.0.43->vllm<=0.6.3->-r requirements.txt (line 13))\n", + " Downloading lark-1.2.2-py3-none-any.whl.metadata (1.8 kB)\n", + "Requirement already satisfied: nest-asyncio in /usr/local/lib/python3.11/dist-packages (from outlines<0.1,>=0.0.43->vllm<=0.6.3->-r requirements.txt (line 13)) (1.6.0)\n", + "Collecting diskcache (from outlines<0.1,>=0.0.43->vllm<=0.6.3->-r requirements.txt (line 13))\n", + " Downloading diskcache-5.6.3-py3-none-any.whl.metadata (20 kB)\n", + "Requirement already satisfied: numba in /usr/local/lib/python3.11/dist-packages (from outlines<0.1,>=0.0.43->vllm<=0.6.3->-r requirements.txt (line 13)) (0.61.0)\n", + "Collecting pycountry (from outlines<0.1,>=0.0.43->vllm<=0.6.3->-r requirements.txt (line 13))\n", + " Downloading pycountry-24.6.1-py3-none-any.whl.metadata (12 kB)\n", + "Collecting pyairports (from outlines<0.1,>=0.0.43->vllm<=0.6.3->-r requirements.txt (line 13))\n", + " Downloading pyairports-2.1.1-py3-none-any.whl.metadata (1.7 kB)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in /usr/local/lib/python3.11/dist-packages (from pydantic>=2.9->vllm<=0.6.3->-r requirements.txt (line 13)) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.27.2 in /usr/local/lib/python3.11/dist-packages (from pydantic>=2.9->vllm<=0.6.3->-r requirements.txt (line 13)) (2.27.2)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/dist-packages (from requests>=2.32.2->datasets->-r requirements.txt (line 3)) (3.4.1)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/dist-packages (from requests>=2.32.2->datasets->-r requirements.txt (line 3)) (3.10)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/dist-packages (from requests>=2.32.2->datasets->-r requirements.txt (line 3)) (2.3.0)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/dist-packages (from requests>=2.32.2->datasets->-r requirements.txt (line 3)) (2025.1.31)\n", + "Requirement already satisfied: zipp>=3.20 in /usr/local/lib/python3.11/dist-packages (from importlib-metadata->vllm<=0.6.3->-r requirements.txt (line 13)) (3.21.0)\n", + "Requirement already satisfied: h11>=0.8 in /usr/local/lib/python3.11/dist-packages (from uvicorn[standard]->vllm<=0.6.3->-r requirements.txt (line 13)) (0.14.0)\n", + "Collecting httptools>=0.6.3 (from uvicorn[standard]->vllm<=0.6.3->-r requirements.txt (line 13))\n", + " Downloading httptools-0.6.4-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.6 kB)\n", + "Collecting python-dotenv>=0.13 (from uvicorn[standard]->vllm<=0.6.3->-r requirements.txt (line 13))\n", + " Downloading python_dotenv-1.0.1-py3-none-any.whl.metadata (23 kB)\n", + "Collecting uvloop!=0.15.0,!=0.15.1,>=0.14.0 (from uvicorn[standard]->vllm<=0.6.3->-r requirements.txt (line 13))\n", + " Downloading uvloop-0.21.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.9 kB)\n", + "Collecting watchfiles>=0.13 (from uvicorn[standard]->vllm<=0.6.3->-r requirements.txt (line 13))\n", + " Downloading watchfiles-1.0.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.9 kB)\n", + "Requirement already satisfied: websockets>=10.4 in /usr/local/lib/python3.11/dist-packages (from uvicorn[standard]->vllm<=0.6.3->-r requirements.txt (line 13)) (14.2)\n", + "Requirement already satisfied: smmap<6,>=3.0.1 in /usr/local/lib/python3.11/dist-packages (from gitdb<5,>=4.0.1->gitpython!=3.1.29,>=1.0.0->wandb->-r requirements.txt (line 14)) (5.0.2)\n", + "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.11/dist-packages (from httpx<1,>=0.23.0->openai>=1.40.0->vllm<=0.6.3->-r requirements.txt (line 13)) (1.0.7)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.11/dist-packages (from jinja2->torch>=2.0.0->accelerate->-r requirements.txt (line 1)) (3.0.2)\n", + "Requirement already satisfied: llvmlite<0.45,>=0.44.0dev0 in /usr/local/lib/python3.11/dist-packages (from numba->outlines<0.1,>=0.0.43->vllm<=0.6.3->-r requirements.txt (line 13)) (0.44.0)\n", + "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.11/dist-packages (from sympy->torch>=2.0.0->accelerate->-r requirements.txt (line 1)) (1.3.0)\n", + "Downloading datasets-3.2.0-py3-none-any.whl (480 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m480.6/480.6 kB\u001b[0m \u001b[31m37.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading dill-0.3.8-py3-none-any.whl (116 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m12.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading ray-2.42.0-cp311-cp311-manylinux2014_x86_64.whl (67.4 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.4/67.4 MB\u001b[0m \u001b[31m31.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading transformers-4.47.1-py3-none-any.whl (10.1 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m10.1/10.1 MB\u001b[0m \u001b[31m110.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading vllm-0.6.3-cp38-abi3-manylinux1_x86_64.whl (193.5 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m193.5/193.5 MB\u001b[0m \u001b[31m6.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading gguf-0.10.0-py3-none-any.whl (71 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.6/71.6 kB\u001b[0m \u001b[31m7.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading lm_format_enforcer-0.10.6-py3-none-any.whl (43 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.7/43.7 kB\u001b[0m \u001b[31m4.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading torchvision-0.19.0-cp311-cp311-manylinux1_x86_64.whl (7.0 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.0/7.0 MB\u001b[0m \u001b[31m83.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading xformers-0.0.27.post2-cp311-cp311-manylinux2014_x86_64.whl (20.8 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m20.8/20.8 MB\u001b[0m \u001b[31m50.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading fastapi-0.115.8-py3-none-any.whl (94 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m94.8/94.8 kB\u001b[0m \u001b[31m9.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading fsspec-2024.9.0-py3-none-any.whl (179 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m179.3/179.3 kB\u001b[0m \u001b[31m15.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading mistral_common-1.5.3-py3-none-any.whl (6.5 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.5/6.5 MB\u001b[0m \u001b[31m125.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading multiprocess-0.70.16-py311-none-any.whl (143 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m143.5/143.5 kB\u001b[0m \u001b[31m14.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading outlines-0.0.46-py3-none-any.whl (101 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m101.9/101.9 kB\u001b[0m \u001b[31m11.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading prometheus_fastapi_instrumentator-7.0.2-py3-none-any.whl (18 kB)\n", + "Downloading tiktoken-0.8.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m56.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading msgspec-0.19.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (210 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m210.7/210.7 kB\u001b[0m \u001b[31m19.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading nvidia_ml_py-12.570.86-py3-none-any.whl (44 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m44.4/44.4 kB\u001b[0m \u001b[31m4.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading partial_json_parser-0.2.1.1.post5-py3-none-any.whl (10 kB)\n", + "Downloading xxhash-3.5.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.8/194.8 kB\u001b[0m \u001b[31m22.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading httptools-0.6.4-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (459 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m459.8/459.8 kB\u001b[0m \u001b[31m41.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading interegular-0.3.3-py37-none-any.whl (23 kB)\n", + "Downloading python_dotenv-1.0.1-py3-none-any.whl (19 kB)\n", + "Downloading starlette-0.45.3-py3-none-any.whl (71 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.5/71.5 kB\u001b[0m \u001b[31m7.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading uvloop-0.21.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.0 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.0/4.0 MB\u001b[0m \u001b[31m97.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading watchfiles-1.0.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (452 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m452.6/452.6 kB\u001b[0m \u001b[31m38.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading diskcache-5.6.3-py3-none-any.whl (45 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.5/45.5 kB\u001b[0m \u001b[31m4.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading lark-1.2.2-py3-none-any.whl (111 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m111.0/111.0 kB\u001b[0m \u001b[31m11.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading pyairports-2.1.1-py3-none-any.whl (371 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m371.7/371.7 kB\u001b[0m \u001b[31m33.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading pycountry-24.6.1-py3-none-any.whl (6.3 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.3/6.3 MB\u001b[0m \u001b[31m127.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hDownloading uvicorn-0.34.0-py3-none-any.whl (62 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.3/62.3 kB\u001b[0m \u001b[31m6.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hBuilding wheels for collected packages: flash-attn\n", + " Building wheel for flash-attn (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for flash-attn: filename=flash_attn-2.7.4.post1-cp311-cp311-linux_x86_64.whl size=187805408 sha256=92cf49e6f66795b6934cec0cba526ed6e45d3313de3f905d45df8773f19092a9\n", + " Stored in directory: /root/.cache/pip/wheels/3d/88/d8/284b89f56af7d5bf366b10d6b8e251ac8a7c7bf3f04203fb4f\n", + "Successfully built flash-attn\n", + "Installing collected packages: pyairports, nvidia-ml-py, xxhash, uvloop, uvicorn, python-dotenv, pycountry, partial-json-parser, msgspec, lark, interegular, httptools, gguf, fsspec, diskcache, dill, watchfiles, tiktoken, starlette, multiprocess, prometheus-fastapi-instrumentator, lm-format-enforcer, fastapi, xformers, transformers, torchvision, ray, mistral-common, flash-attn, datasets, outlines, vllm\n", + " Attempting uninstall: fsspec\n", + " Found existing installation: fsspec 2024.10.0\n", + " Uninstalling fsspec-2024.10.0:\n", + " Successfully uninstalled fsspec-2024.10.0\n", + " Attempting uninstall: dill\n", + " Found existing installation: dill 0.3.9\n", + " Uninstalling dill-0.3.9:\n", + " Successfully uninstalled dill-0.3.9\n", + " Attempting uninstall: transformers\n", + " Found existing installation: transformers 4.48.2\n", + " Uninstalling transformers-4.48.2:\n", + " Successfully uninstalled transformers-4.48.2\n", + " Attempting uninstall: torchvision\n", + " Found existing installation: torchvision 0.20.1+cu124\n", + " Uninstalling torchvision-0.20.1+cu124:\n", + " Successfully uninstalled torchvision-0.20.1+cu124\n", + "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n", + "gcsfs 2024.10.0 requires fsspec==2024.10.0, but you have fsspec 2024.9.0 which is incompatible.\n", + "torchaudio 2.5.1+cu124 requires torch==2.5.1, but you have torch 2.4.0 which is incompatible.\u001b[0m\u001b[31m\n", + "\u001b[0mSuccessfully installed datasets-3.2.0 dill-0.3.8 diskcache-5.6.3 fastapi-0.115.8 flash-attn-2.7.4.post1 fsspec-2024.9.0 gguf-0.10.0 httptools-0.6.4 interegular-0.3.3 lark-1.2.2 lm-format-enforcer-0.10.6 mistral-common-1.5.3 msgspec-0.19.0 multiprocess-0.70.16 nvidia-ml-py-12.570.86 outlines-0.0.46 partial-json-parser-0.2.1.1.post5 prometheus-fastapi-instrumentator-7.0.2 pyairports-2.1.1 pycountry-24.6.1 python-dotenv-1.0.1 ray-2.42.0 starlette-0.45.3 tiktoken-0.8.0 torchvision-0.19.0 transformers-4.47.1 uvicorn-0.34.0 uvloop-0.21.0 vllm-0.6.3 watchfiles-1.0.4 xformers-0.0.27.post2 xxhash-3.5.0\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [], + "metadata": { + "id": "ACn66BcVoeCi" + } + }, + { + "cell_type": "markdown", + "source": [ + "### Enable Metrics Platform (WandB.ai)" + ], + "metadata": { + "id": "A2yLwOhBYnRa" + } + }, + { + "cell_type": "code", + "source": [ + "import wandb\n", + "wandb.login()" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 193 + }, + "id": "tWdrVq4lILbg", + "outputId": "e4b56224-f206-48d3-cfee-61066e02f26a", + "collapsed": true + }, + "execution_count": 1, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "application/javascript": [ + "\n", + " window._wandbApiKey = new Promise((resolve, reject) => {\n", + " function loadScript(url) {\n", + " return new Promise(function(resolve, reject) {\n", + " let newScript = document.createElement(\"script\");\n", + " newScript.onerror = reject;\n", + " newScript.onload = resolve;\n", + " document.body.appendChild(newScript);\n", + " newScript.src = url;\n", + " });\n", + " }\n", + " loadScript(\"https://cdn.jsdelivr.net/npm/postmate/build/postmate.min.js\").then(() => {\n", + " const iframe = document.createElement('iframe')\n", + " iframe.style.cssText = \"width:0;height:0;border:none\"\n", + " document.body.appendChild(iframe)\n", + " const handshake = new Postmate({\n", + " container: iframe,\n", + " url: 'https://wandb.ai/authorize'\n", + " });\n", + " const timeout = setTimeout(() => reject(\"Couldn't auto authenticate\"), 5000)\n", + " handshake.then(function(child) {\n", + " child.on('authorize', data => {\n", + " clearTimeout(timeout)\n", + " resolve(data)\n", + " });\n", + " });\n", + " })\n", + " });\n", + " " + ] + }, + "metadata": {} + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Logging into wandb.ai. (Learn how to deploy a W&B server locally: https://wandb.me/wandb-server)\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: You can find your API key in your browser here: https://wandb.ai/authorize\n", + "wandb: Paste an API key from your profile and hit enter:" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + " ··········\n" + ] + }, + { + "output_type": "stream", + "name": "stderr", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m If you're specifying your api key in code, ensure this code is not shared publicly.\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Consider setting the WANDB_API_KEY environment variable, or running `wandb login` from the command line.\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n", + "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mmrwulf-cleaner\u001b[0m (\u001b[33mmrwulf-cleaner-my\u001b[0m) to \u001b[32mhttps://api.wandb.ai\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n" + ] + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "True" + ] + }, + "metadata": {}, + "execution_count": 1 + } + ] + }, + { + "cell_type": "code", + "source": [ + "wandb.init(project=\"Project1\", config={\"some_key\": \"some_value\"})" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 142 + }, + "id": "75sTMqZIJKRq", + "outputId": "4d5a22f4-b403-49ca-ebe8-38a195495e0f", + "collapsed": true + }, + "execution_count": 2, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "\u001b[34m\u001b[1mwandb\u001b[0m: Using wandb-core as the SDK backend. Please refer to https://wandb.me/wandb-core for more information.\n" + ] + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "Tracking run with wandb version 0.19.6" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "Run data is saved locally in /content/wandb/run-20250210_081136-9h4rwllw" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "Syncing run clear-eon-5 to Weights & Biases (docs)
" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + " View project at https://wandb.ai/mrwulf-cleaner-my/Project1" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + "" + ], + "text/html": [ + " View run at https://wandb.ai/mrwulf-cleaner-my/Project1/runs/9h4rwllw" + ] + }, + "metadata": {} + }, + { + "output_type": "execute_result", + "data": { + "text/html": [ + "" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "execution_count": 2 + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "# RL" + ], + "metadata": { + "id": "maKAyYyJpT5M" + } + }, + { + "cell_type": "markdown", + "source": [ + "## Qwen2.5-3B" + ], + "metadata": { + "id": "jxz10dUNqfoM" + } + }, + { + "cell_type": "markdown", + "source": [ + "### Konfektioniere das Countdown Dataset for Model Qwen2.5-3B" + ], + "metadata": { + "id": "rXHkrMKfYzBw" + } + }, + { + "cell_type": "code", + "source": [ + "!python TinyZero/examples/data_preprocess/countdown.py --local_dir=./content/data" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "FmfrWspLSTMN", + "outputId": "7a5c2a2c-53f5-4e2d-c53d-c2582d474935", + "collapsed": true + }, + "execution_count": 3, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "README.md: 100% 314/314 [00:00<00:00, 2.67MB/s]\n", + "train-00000-of-00001.parquet: 100% 2.85M/2.85M [00:00<00:00, 43.9MB/s]\n", + "Generating train split: 100% 490364/490364 [00:00<00:00, 3042206.89 examples/s]\n", + "Map: 100% 327680/327680 [00:18<00:00, 18125.99 examples/s]\n", + "Map: 100% 1024/1024 [00:00<00:00, 15078.16 examples/s]\n", + "Creating parquet from Arrow format: 100% 328/328 [00:00<00:00, 547.55ba/s]\n", + "Creating parquet from Arrow format: 100% 2/2 [00:00<00:00, 710.60ba/s]\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "### Download Model Qwen2.5-3B\n" + ], + "metadata": { + "id": "FS9J3e3wZBzt" + } + }, + { + "cell_type": "code", + "source": [ + "!huggingface-cli download Qwen/Qwen2.5-3B --local-dir $HOME/models/Qwen2.5-3B" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "collapsed": true, + "id": "iKhVyxVXanDT", + "outputId": "0c94376a-9199-4076-a051-bfa70069739e" + }, + "execution_count": 4, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "\rFetching 12 files: 0% 0/12 [00:00system\n", + "You are a hel... math \n", + "1 [{'content': '<|im_start|>system\n", + "You are a hel... math \n", + "2 [{'content': '<|im_start|>system\n", + "You are a hel... math \n", + "3 [{'content': '<|im_start|>system\n", + "You are a hel... math \n", + "4 [{'content': '<|im_start|>system\n", + "You are a hel... math \n", + "\n", + " reward_model \\\n", + "0 {'ground_truth': {'numbers': [79, 17, 60], 'ta... \n", + "1 {'ground_truth': {'numbers': [11, 34, 82, 80],... \n", + "2 {'ground_truth': {'numbers': [51, 4, 60, 35], ... \n", + "3 {'ground_truth': {'numbers': [34, 98, 1, 96], ... \n", + "4 {'ground_truth': {'numbers': [46, 9, 49, 56], ... \n", + "\n", + " extra_info \n", + "0 {'index': 0, 'split': 'test'} \n", + "1 {'index': 1, 'split': 'test'} \n", + "2 {'index': 2, 'split': 'test'} \n", + "3 {'index': 3, 'split': 'test'} \n", + "4 {'index': 4, 'split': 'test'} \n", + "(1024, 7)\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "!bash ./TinyZero/scripts/train_tiny_zero.sh" + ], + "metadata": { + "id": "1SQfqMz0gjBw", + "colab": { + "base_uri": "https://localhost:8080/" + }, + "collapsed": true, + "outputId": "a22a1da2-e3ad-441c-c00a-1d28aeb29088" + }, + "execution_count": 8, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "The cache for model files in Transformers v4.22.0 has been updated. Migrating your old cache. This is a one-time only operation. You can interrupt this and resume the migration later on by calling `transformers.utils.move_cache()`.\n", + "\r0it [00:00, ?it/s]\r0it [00:00, ?it/s]\n", + "2025-02-10 08:58:56,003\tINFO worker.py:1841 -- Started a local Ray instance.\n", + "\u001b[36m(main_task pid=29102)\u001b[0m {'actor_rollout_ref': {'actor': {'clip_ratio': 0.2,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'entropy_coeff': 0.001,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'fsdp_config': {'grad_offload': False,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'optimizer_offload': False,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'param_offload': False,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'wrap_policy': {'min_num_params': 0}},\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'grad_clip': 1.0,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'optim': {'lr': 1e-06,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'lr_warmup_steps_ratio': 0.0,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'min_lr_ratio': None,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'total_training_steps': -1,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'warmup_style': 'constant'},\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'ppo_epochs': 1,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'ppo_micro_batch_size': 8,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'ppo_mini_batch_size': 128,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'shuffle': True,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'strategy': 'fsdp'},\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'hybrid_engine': True,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'model': {'enable_gradient_checkpointing': False,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'external_lib': None,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'override_config': {},\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'path': '/root/models/Qwen2.5-3B-Instruct'},\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'ref': {'fsdp_config': {'param_offload': False,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'wrap_policy': {'min_num_params': 0}},\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'log_prob_micro_batch_size': 4},\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'rollout': {'do_sample': True,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'dtype': 'bfloat16',\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'enforce_eager': True,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'free_cache_engine': True,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'gpu_memory_utilization': 0.4,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'ignore_eos': False,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'load_format': 'dummy_dtensor',\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'log_prob_micro_batch_size': 8,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'max_num_batched_tokens': 8192,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'max_num_seqs': 1024,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'name': 'vllm',\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'prompt_length': 256,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'response_length': 1024,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'temperature': 1.0,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'tensor_model_parallel_size': 2,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'top_k': -1,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'top_p': 1}},\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'algorithm': {'adv_estimator': 'gae',\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'gamma': 1.0,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'kl_ctrl': {'kl_coef': 0.001, 'type': 'fixed'},\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'kl_penalty': 'kl',\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'lam': 1.0},\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'critic': {'cliprange_value': 0.5,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'grad_clip': 1.0,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'model': {'enable_gradient_checkpointing': False,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'external_lib': None,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'fsdp_config': {'grad_offload': False,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'optimizer_offload': False,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'param_offload': False,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'wrap_policy': {'min_num_params': 0}},\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'override_config': {},\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'path': '/root/models/Qwen2.5-3B-Instruct',\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'tokenizer_path': '/root/models/Qwen2.5-3B-Instruct'},\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'optim': {'lr': 1e-05,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'lr_warmup_steps_ratio': 0.0,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'min_lr_ratio': None,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'total_training_steps': -1,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'warmup_style': 'constant'},\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'ppo_epochs': 1,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'ppo_micro_batch_size': 8,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'ppo_mini_batch_size': 128,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'shuffle': True,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'strategy': 'fsdp'},\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'data': {'max_prompt_length': 256,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'max_response_length': 1024,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'prompt_key': 'prompt',\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'return_raw_chat': False,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'return_raw_input_ids': False,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'tokenizer': None,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'train_batch_size': 256,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'train_files': '/content/content/data/train.parquet',\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'val_batch_size': 256,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'val_files': '/content/content/data/test.parquet'},\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'reward_model': {'enable': False,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'max_length': None,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'micro_batch_size': 64,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'model': {'external_lib': None,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'fsdp_config': {'min_num_params': 0,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'param_offload': False},\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'input_tokenizer': '/root/models/Qwen2.5-3B-Instruct',\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'path': '~/models/FsfairX-LLaMA3-RM-v0.1'},\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'strategy': 'fsdp'},\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'trainer': {'critic_warmup': 0,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'default_hdfs_dir': None,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'default_local_dir': 'checkpoints/TinyZero/countdown-qwen2.5-3b-instruct',\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'experiment_name': 'countdown-qwen2.5-3b-instruct',\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'logger': ['wandb'],\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'n_gpus_per_node': 2,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'nnodes': 1,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'project_name': 'TinyZero',\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'save_freq': 100,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'test_freq': 100,\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'total_epochs': 15,\u001b[36m(main_task pid=29102)\u001b[0m 2025-02-10 08:59:02.978050: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:477] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "\u001b[36m(main_task pid=29102)\u001b[0m WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n", + "\u001b[36m(main_task pid=29102)\u001b[0m E0000 00:00:1739177942.997757 29102 cuda_dnn.cc:8310] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "\u001b[36m(main_task pid=29102)\u001b[0m E0000 00:00:1739177943.003847 29102 cuda_blas.cc:1418] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 2025-02-10 08:59:03.023493: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", + "\u001b[36m(main_task pid=29102)\u001b[0m To enable the following instructions: AVX2 AVX512F FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", + "\n", + "\u001b[36m(main_task pid=29102)\u001b[0m 'val_before_train': False}}\n", + "\u001b[36m(main_task pid=29102)\u001b[0m original dataset len: 327680\n", + "\u001b[36m(main_task pid=29102)\u001b[0m filter dataset len: 327680\n", + "\u001b[36m(main_task pid=29102)\u001b[0m original dataset len: 1024\n", + "\u001b[36m(main_task pid=29102)\u001b[0m filter dataset len: 1024\n", + "\u001b[36m(main_task pid=29102)\u001b[0m Size of train dataloader: 1280\n", + "\u001b[36m(main_task pid=29102)\u001b[0m Size of val dataloader: 4\n", + "\u001b[36m(autoscaler +3m26s)\u001b[0m Tip: use `ray status` to view detailed cluster status. To disable these messages, set RAY_SCHEDULER_EVENTS=0.\n", + "\u001b[33m(autoscaler +3m26s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4m1s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4m36s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5m11s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5m46s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6m22s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6m57s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +7m32s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +8m7s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +8m42s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +9m17s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +9m52s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +10m27s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +11m2s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +11m37s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +12m12s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +12m47s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +13m22s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +13m57s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +14m32s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +15m7s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +15m42s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +16m17s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +16m52s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +17m27s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +18m2s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +18m37s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +19m12s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +19m47s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +20m22s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +20m57s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +21m32s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +22m7s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +22m42s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +23m17s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +23m52s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +24m27s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +25m3s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +25m38s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +26m13s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +26m48s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +27m23s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +27m58s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +28m33s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +29m8s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +29m43s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +30m18s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +30m53s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +31m28s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +32m3s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +32m38s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +33m13s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +33m48s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +34m23s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +34m58s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +35m33s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +36m8s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +36m43s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +37m18s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +37m53s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +38m28s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +39m3s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +39m38s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +40m13s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +40m48s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +41m23s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +41m58s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +42m33s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +43m9s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +43m43s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +44m19s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +44m54s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +45m29s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +46m4s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +46m39s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +47m14s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +47m49s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +48m24s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +48m59s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +49m34s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +50m9s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +50m44s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +51m19s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +51m54s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +52m29s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +53m4s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +53m39s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +54m14s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +54m49s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +55m24s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +55m59s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +56m34s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +57m9s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +57m44s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +58m19s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +58m54s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +59m29s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h4s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h39s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h1m14s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h1m50s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h2m24s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h3m0s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h3m35s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h4m10s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h4m45s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h5m20s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h5m55s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h6m30s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h7m5s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h7m40s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h8m15s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h8m50s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h9m25s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h10m0s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h10m35s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h11m10s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h11m45s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h12m20s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h12m55s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h13m30s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h14m5s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h14m40s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h15m15s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h15m50s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h16m25s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h17m0s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h17m35s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h18m10s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h18m45s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h19m20s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h19m55s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h20m31s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h21m5s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h21m41s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h22m16s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h22m51s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h23m26s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h24m1s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h24m36s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h25m11s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h25m46s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h26m21s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h26m56s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h27m31s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h28m6s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h28m41s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h29m16s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h29m51s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h30m26s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h31m1s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h31m36s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h32m11s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h32m46s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h33m21s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h33m56s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h34m31s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h35m6s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h35m41s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h36m16s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h36m51s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h37m26s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h38m1s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h38m36s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h39m12s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h39m46s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h40m22s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h40m57s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h41m32s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h42m7s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h42m42s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h43m17s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h43m52s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h44m27s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h45m2s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h45m37s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h46m12s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h46m47s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h47m22s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h47m57s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h48m32s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h49m7s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h49m42s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h50m17s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h50m52s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h51m27s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h52m2s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h52m37s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h53m12s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h53m47s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h54m22s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h54m57s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h55m32s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h56m7s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h56m42s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h57m17s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h57m52s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h58m27s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h59m3s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +1h59m38s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h13s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h48s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h1m23s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h1m58s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h2m33s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h3m8s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h3m43s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h4m18s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h4m53s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h5m28s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h6m3s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h6m38s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h7m13s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h7m48s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h8m23s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h8m58s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h9m33s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h10m8s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h10m43s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h11m18s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h11m53s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h12m28s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h13m3s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h13m38s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h14m13s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h14m48s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h15m23s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h15m58s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h16m33s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h17m9s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h17m43s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h18m19s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h18m54s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h19m29s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h20m4s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h20m39s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h21m14s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h21m49s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h22m24s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h22m59s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h23m34s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h24m9s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h24m44s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h25m19s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h25m54s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h26m29s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h27m4s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h27m39s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h28m14s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h28m49s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h29m24s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h29m59s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h30m34s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h31m9s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h31m44s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h32m19s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h32m54s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h33m29s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h34m4s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h34m39s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h35m14s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h35m49s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h36m25s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h37m0s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h37m35s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h38m10s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h38m45s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h39m20s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h39m55s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h40m30s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h41m5s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h41m40s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h42m15s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h42m50s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h43m25s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h44m0s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h44m35s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h45m10s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h45m45s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h46m20s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h46m55s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h47m30s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h48m5s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h48m40s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h49m15s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h49m50s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h50m25s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h51m0s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h51m35s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h52m10s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h52m45s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h53m20s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h53m55s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h54m30s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h55m6s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h55m41s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h56m16s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h56m51s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h57m26s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h58m1s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h58m36s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h59m11s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +2h59m46s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h21s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h56s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h1m31s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h2m6s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h2m41s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h3m16s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h3m51s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h4m26s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h5m1s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h5m36s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h6m11s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h6m46s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h7m21s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h7m56s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h8m31s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h9m6s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h9m41s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h10m16s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h10m51s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h11m26s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h12m1s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h12m36s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h13m11s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h13m47s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h14m22s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h14m57s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h15m32s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h16m7s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h16m42s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h17m17s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h17m52s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h18m27s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h19m2s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h19m37s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h20m12s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h20m47s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h21m22s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h21m57s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h22m32s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h23m7s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h23m42s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h24m17s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h24m52s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h25m27s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h26m2s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h26m37s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h27m12s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h27m47s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h28m22s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h28m57s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h29m32s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h30m7s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h30m42s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h31m17s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h31m52s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h32m28s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h33m3s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h33m38s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h34m13s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h34m48s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h35m23s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h35m58s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h36m33s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h37m8s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h37m43s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h38m18s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h38m53s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h39m28s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h40m3s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h40m38s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h41m13s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h41m48s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h42m23s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h42m58s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h43m33s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h44m8s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h44m43s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h45m18s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h45m53s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h46m28s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h47m3s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h47m38s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h48m13s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h48m48s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h49m23s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h49m58s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h50m34s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h51m8s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h51m44s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h52m19s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h52m54s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h53m29s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h54m4s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h54m39s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h55m14s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h55m49s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h56m24s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h56m59s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h57m34s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h58m9s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h58m44s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h59m19s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +3h59m54s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h29s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h1m4s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h1m39s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h2m14s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h2m49s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h3m24s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h3m59s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h4m34s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h5m9s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h5m44s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h6m19s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h6m54s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h7m29s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h8m4s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h8m39s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h9m14s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h9m49s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h10m25s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h11m0s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h11m35s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h12m10s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h12m45s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h13m20s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h13m55s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h14m30s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h15m5s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h15m40s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h16m15s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h16m50s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h17m25s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h18m0s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h18m35s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h19m10s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h19m45s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h20m20s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h20m55s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h21m30s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h22m5s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h22m40s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h23m15s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h23m50s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h24m25s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h25m0s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h25m35s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h26m10s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h26m45s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h27m20s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h27m55s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h28m30s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h29m6s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h29m41s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h30m16s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h30m51s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h31m26s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h32m1s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h32m36s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h33m11s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h33m46s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h34m21s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h34m56s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h35m31s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h36m6s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h36m41s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h37m16s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h37m51s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h38m26s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h39m1s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h39m36s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h40m11s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h40m46s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h41m21s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h41m56s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h42m31s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h43m6s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h43m41s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h44m16s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h44m51s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h45m26s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h46m1s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h46m36s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h47m11s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h47m47s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h48m22s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h48m57s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h49m32s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h50m7s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h50m42s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h51m17s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h51m52s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h52m27s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h53m2s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h53m37s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h54m12s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h54m47s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h55m22s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h55m57s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h56m32s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h57m7s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h57m42s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h58m17s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h58m52s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +4h59m27s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h2s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h37s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h1m12s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h1m47s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h2m22s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h2m57s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h3m32s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h4m7s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h4m42s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h5m17s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h5m52s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h6m27s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h7m3s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h7m38s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h8m13s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h8m48s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h9m23s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h9m58s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h10m33s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h11m8s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h11m43s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h12m18s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h12m53s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h13m28s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h14m3s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h14m38s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h15m13s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h15m48s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h16m23s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h16m58s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h17m33s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h18m8s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h18m43s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h19m18s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h19m53s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h20m28s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h21m3s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h21m38s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h22m13s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h22m48s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h23m23s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h23m58s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h24m33s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h25m9s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h25m43s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h26m19s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h26m54s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h27m29s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h28m4s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h28m39s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h29m14s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h29m49s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h30m24s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h30m59s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h31m34s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h32m9s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h32m44s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h33m19s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h33m54s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h34m29s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h35m4s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h35m39s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h36m14s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h36m49s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h37m24s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h37m59s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h38m34s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h39m9s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h39m44s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h40m19s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h40m54s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h41m29s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h42m4s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h42m39s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h43m14s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h43m49s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h44m25s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h45m0s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h45m35s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h46m10s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h46m45s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h47m20s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h47m55s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h48m30s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h49m5s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h49m40s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h50m15s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h50m50s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h51m25s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h52m0s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h52m35s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h53m10s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h53m45s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h54m20s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h54m55s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h55m30s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h56m5s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h56m40s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h57m15s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h57m50s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h58m25s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h59m0s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +5h59m35s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h10s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h45s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h1m20s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h1m55s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h2m30s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h3m6s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h3m40s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h4m16s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h4m51s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h5m26s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h6m1s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h6m36s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h7m11s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h7m46s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h8m21s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h8m56s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h9m31s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h10m6s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h10m41s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h11m16s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h11m51s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h12m26s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h13m1s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h13m36s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h14m11s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h14m46s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h15m21s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h15m56s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h16m31s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h17m6s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h17m41s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h18m16s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h18m51s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h19m26s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h20m1s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h20m36s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h21m11s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h21m46s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h22m22s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h22m57s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h23m32s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h24m7s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h24m42s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h25m17s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h25m52s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h26m27s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h27m2s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h27m37s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h28m12s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h28m47s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h29m22s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h29m57s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h30m32s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h31m7s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h31m42s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h32m17s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h32m52s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h33m27s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h34m2s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h34m37s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h35m12s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h35m47s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h36m22s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h36m57s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h37m32s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h38m7s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h38m42s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h39m17s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h39m52s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h40m28s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h41m3s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h41m38s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h42m13s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h42m48s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h43m23s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h43m58s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h44m33s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h45m8s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h45m43s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h46m18s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h46m53s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h47m28s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h48m3s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h48m38s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h49m13s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h49m48s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h50m23s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h50m58s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h51m33s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h52m8s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue.\n", + "\u001b[33m(autoscaler +6h52m43s)\u001b[0m Error: No available node types can fulfill resource request defaultdict(, {'CPU': 2.0, 'GPU': 2.0}). Add suitable node types to this cluster to resolve this issue." + ] + } + ] + } + ] +} \ No newline at end of file