20230328 public 0.8955

ML2023_hw04.ipynb

@@ -68,7 +68,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 1,
    "metadata": {
     "id": "E6burzCXIyuA"
    },
@@ -129,7 +129,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 2,
    "metadata": {
     "id": "KpuGxl4CI2pr"
    },
@@ -158,7 +158,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 3,
    "metadata": {
     "id": "B7c2gZYoJDRS"
    },
@@ -234,7 +234,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 4,
    "metadata": {
     "id": "iXZ5B0EKJGs8"
    },
@@ -254,7 +254,7 @@
     "        #   Change Transformer to Conformer.\n",
     "        #   https://arxiv.org/abs/2005.08100\n",
     "        self.encoder_layer = nn.TransformerEncoderLayer(\n",
-    "            d_model=d_model, dim_feedforward=256, nhead=4\n",
+    "            d_model=d_model, dim_feedforward=256, nhead=8\n",
     "        )\n",
     "        self.encoder = nn.TransformerEncoder(self.encoder_layer, num_layers=2)\n",
     "\n",
@@ -304,7 +304,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 5,
    "metadata": {
     "id": "ykt0N1nVJJi2"
    },
@@ -372,7 +372,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 6,
    "metadata": {
     "id": "N-rr8529JMz0"
    },
@@ -412,7 +412,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 7,
    "metadata": {
     "id": "YAiv6kpdJRTJ"
    },
@@ -481,156 +481,352 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Train: 100% 2000/2000 [02:33<00:00, 13.01 step/s, accuracy=0.03, loss=4.87, step=2000]\n",
-      "Valid: 100% 5664/5667 [00:49<00:00, 114.83 uttr/s, accuracy=0.06, loss=4.86] \n",
-      "Train: 100% 2000/2000 [02:31<00:00, 13.22 step/s, accuracy=0.12, loss=4.36, step=4000]  \n",
-      "Valid: 100% 5664/5667 [00:49<00:00, 113.88 uttr/s, accuracy=0.12, loss=4.32] \n",
-      "Train: 100% 2000/2000 [02:29<00:00, 13.37 step/s, accuracy=0.25, loss=3.72, step=6000] \n",
-      "Valid: 100% 5664/5667 [00:50<00:00, 112.95 uttr/s, accuracy=0.16, loss=4.03] \n",
-      "Train: 100% 2000/2000 [03:16<00:00, 10.20 step/s, accuracy=0.31, loss=3.32, step=8000] \n",
-      "Valid: 100% 5664/5667 [00:49<00:00, 115.03 uttr/s, accuracy=0.20, loss=3.76] \n",
-      "Train: 100% 2000/2000 [02:31<00:00, 13.20 step/s, accuracy=0.22, loss=3.87, step=1e+4]\n",
-      "Valid: 100% 5664/5667 [00:49<00:00, 113.42 uttr/s, accuracy=0.22, loss=3.69] \n",
-      "Train:   0% 3/2000 [00:00<02:22, 14.05 step/s, accuracy=0.19, loss=3.41, step=1e+4]"
+      "Train: 100% 2000/2000 [03:09<00:00, 10.54 step/s, accuracy=0.59, loss=1.64, step=2000]\n",
+      "Valid: 100% 5664/5667 [00:50<00:00, 111.30 uttr/s, accuracy=0.61, loss=1.70] \n",
+      "Train: 100% 2000/2000 [03:06<00:00, 10.70 step/s, accuracy=0.62, loss=1.42, step=4000]  \n",
+      "Valid: 100% 5664/5667 [00:51<00:00, 109.66 uttr/s, accuracy=0.61, loss=1.70] \n",
+      "Train: 100% 2000/2000 [03:09<00:00, 10.58 step/s, accuracy=0.75, loss=1.17, step=6000] \n",
+      "Valid: 100% 5664/5667 [00:53<00:00, 106.36 uttr/s, accuracy=0.61, loss=1.73] \n",
+      "Train: 100% 2000/2000 [03:46<00:00,  8.81 step/s, accuracy=0.75, loss=1.15, step=8000] \n",
+      "Valid: 100% 5664/5667 [00:50<00:00, 112.63 uttr/s, accuracy=0.61, loss=1.74] \n",
+      "Train: 100% 2000/2000 [03:01<00:00, 11.02 step/s, accuracy=0.62, loss=1.49, step=1e+4]\n",
+      "Valid: 100% 5664/5667 [00:51<00:00, 109.64 uttr/s, accuracy=0.61, loss=1.71] \n",
+      "Train:   0% 2/2000 [00:00<04:53,  6.81 step/s, accuracy=0.44, loss=2.35, step=1e+4]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Step 10000, best model saved. (accuracy=0.6148)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Train: 100% 2000/2000 [06:48<00:00,  4.90 step/s, accuracy=0.75, loss=1.41, step=12000]   \n",
+      "Valid: 100% 5664/5667 [16:06<00:00,  5.86 uttr/s, accuracy=0.61, loss=1.70]  \n",
+      "Train: 100% 2000/2000 [02:56<00:00, 11.32 step/s, accuracy=0.62, loss=1.82, step=14000] \n",
+      "Valid: 100% 5664/5667 [00:50<00:00, 112.22 uttr/s, accuracy=0.62, loss=1.67] \n",
+      "Train: 100% 2000/2000 [53:33<00:00,  1.61s/ step, accuracy=0.62, loss=1.24, step=16000]    \n",
+      "Valid: 100% 5664/5667 [00:49<00:00, 113.71 uttr/s, accuracy=0.61, loss=1.69] \n",
+      "Train: 100% 2000/2000 [06:01<00:00,  5.53 step/s, accuracy=0.66, loss=1.28, step=18000]  \n",
+      "Valid: 100% 5664/5667 [00:51<00:00, 110.69 uttr/s, accuracy=0.60, loss=1.71] \n",
+      "Train: 100% 2000/2000 [03:03<00:00, 10.87 step/s, accuracy=0.75, loss=1.24, step=2e+4]   \n",
+      "Valid: 100% 5664/5667 [00:50<00:00, 111.59 uttr/s, accuracy=0.63, loss=1.61] \n",
+      "Train:   0% 2/2000 [00:00<04:40,  7.12 step/s, accuracy=0.72, loss=0.98, step=2e+4]"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Step 10000, best model saved. (accuracy=0.2161)\n"
+      "Step 20000, best model saved. (accuracy=0.6289)\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Train: 100% 2000/2000 [02:31<00:00, 13.23 step/s, accuracy=0.31, loss=3.37, step=12000]  \n",
-      "Valid: 100% 5664/5667 [00:50<00:00, 112.45 uttr/s, accuracy=0.25, loss=3.50] \n",
-      "Train: 100% 2000/2000 [02:28<00:00, 13.46 step/s, accuracy=0.12, loss=4.07, step=14000] \n",
-      "Valid: 100% 5664/5667 [00:50<00:00, 112.36 uttr/s, accuracy=0.26, loss=3.41] \n",
-      "Train: 100% 2000/2000 [03:11<00:00, 10.43 step/s, accuracy=0.28, loss=3.16, step=16000] \n",
-      "Valid: 100% 5664/5667 [00:49<00:00, 114.11 uttr/s, accuracy=0.28, loss=3.32] \n",
-      "Train: 100% 2000/2000 [02:30<00:00, 13.32 step/s, accuracy=0.25, loss=3.28, step=18000]\n",
-      "Valid: 100% 5664/5667 [00:50<00:00, 112.56 uttr/s, accuracy=0.32, loss=3.15] \n",
-      "Train: 100% 2000/2000 [02:31<00:00, 13.18 step/s, accuracy=0.34, loss=2.88, step=2e+4]   \n",
-      "Valid: 100% 5664/5667 [00:50<00:00, 111.67 uttr/s, accuracy=0.31, loss=3.13] \n",
-      "Train:   0% 3/2000 [00:00<02:17, 14.50 step/s, accuracy=0.38, loss=2.87, step=2e+4]"
+      "Train: 100% 2000/2000 [03:03<00:00, 10.92 step/s, accuracy=0.62, loss=1.45, step=22000] \n",
+      "Valid: 100% 5664/5667 [00:51<00:00, 110.00 uttr/s, accuracy=0.62, loss=1.64] \n",
+      "Train: 100% 2000/2000 [03:48<00:00,  8.75 step/s, accuracy=0.47, loss=2.43, step=24000] \n",
+      "Valid: 100% 5664/5667 [00:50<00:00, 112.45 uttr/s, accuracy=0.44, loss=2.66] \n",
+      "Train: 100% 2000/2000 [03:03<00:00, 10.92 step/s, accuracy=0.78, loss=1.01, step=26000]\n",
+      "Valid: 100% 5664/5667 [00:50<00:00, 111.25 uttr/s, accuracy=0.63, loss=1.61] \n",
+      "Train: 100% 2000/2000 [03:07<00:00, 10.66 step/s, accuracy=0.72, loss=1.11, step=28000]  \n",
+      "Valid: 100% 5664/5667 [00:51<00:00, 109.39 uttr/s, accuracy=0.62, loss=1.64] \n",
+      "Train: 100% 2000/2000 [03:07<00:00, 10.68 step/s, accuracy=0.78, loss=0.81, step=3e+4]  \n",
+      "Valid: 100% 5664/5667 [00:52<00:00, 108.07 uttr/s, accuracy=0.65, loss=1.57] \n",
+      "Train:   0% 2/2000 [00:00<04:39,  7.16 step/s, accuracy=0.81, loss=0.63, step=3e+4]"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Step 20000, best model saved. (accuracy=0.3169)\n"
+      "Step 30000, best model saved. (accuracy=0.6457)\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Train: 100% 2000/2000 [02:31<00:00, 13.21 step/s, accuracy=0.31, loss=3.03, step=22000] \n",
-      "Valid: 100% 5664/5667 [00:50<00:00, 111.74 uttr/s, accuracy=0.33, loss=3.02] \n",
-      "Train: 100% 2000/2000 [03:20<00:00, 10.00 step/s, accuracy=0.31, loss=2.97, step=24000] \n",
-      "Valid: 100% 5664/5667 [00:49<00:00, 114.55 uttr/s, accuracy=0.34, loss=2.96] \n",
-      "Train: 100% 2000/2000 [02:32<00:00, 13.12 step/s, accuracy=0.50, loss=2.47, step=26000]\n",
-      "Valid: 100% 5664/5667 [00:50<00:00, 112.75 uttr/s, accuracy=0.37, loss=2.89] \n",
-      "Train: 100% 2000/2000 [02:30<00:00, 13.28 step/s, accuracy=0.41, loss=2.66, step=28000]  \n",
-      "Valid: 100% 5664/5667 [00:50<00:00, 111.88 uttr/s, accuracy=0.38, loss=2.80] \n",
-      "Train: 100% 2000/2000 [02:28<00:00, 13.46 step/s, accuracy=0.47, loss=2.04, step=3e+4]  \n",
-      "Valid: 100% 5664/5667 [00:51<00:00, 110.57 uttr/s, accuracy=0.39, loss=2.75] \n",
-      "Train:   0% 3/2000 [00:00<02:29, 13.39 step/s, accuracy=0.53, loss=2.14, step=3e+4]"
+      "Train: 100% 2000/2000 [03:50<00:00,  8.70 step/s, accuracy=0.84, loss=0.77, step=32000] \n",
+      "Valid: 100% 5664/5667 [00:50<00:00, 111.98 uttr/s, accuracy=0.64, loss=1.59] \n",
+      "Train: 100% 2000/2000 [03:03<00:00, 10.87 step/s, accuracy=0.69, loss=1.25, step=34000]  \n",
+      "Valid: 100% 5664/5667 [00:51<00:00, 110.25 uttr/s, accuracy=0.63, loss=1.58] \n",
+      "Train: 100% 2000/2000 [03:00<00:00, 11.05 step/s, accuracy=0.50, loss=1.50, step=36000]  \n",
+      "Valid: 100% 5664/5667 [00:51<00:00, 109.65 uttr/s, accuracy=0.64, loss=1.56] \n",
+      "Train: 100% 2000/2000 [03:03<00:00, 10.89 step/s, accuracy=0.62, loss=1.40, step=38000] \n",
+      "Valid: 100% 5664/5667 [00:52<00:00, 107.96 uttr/s, accuracy=0.64, loss=1.60] \n",
+      "Train: 100% 2000/2000 [09:52<00:00,  3.38 step/s, accuracy=0.62, loss=1.35, step=4e+4]   \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 101.16 uttr/s, accuracy=0.64, loss=1.52]\n",
+      "Train:   0% 0/2000 [00:00<?, ? step/s]"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Step 30000, best model saved. (accuracy=0.3895)\n"
+      "Step 40000, best model saved. (accuracy=0.6457)\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Train: 100% 2000/2000 [03:13<00:00, 10.32 step/s, accuracy=0.59, loss=2.26, step=32000] \n",
-      "Valid: 100% 5664/5667 [00:49<00:00, 115.22 uttr/s, accuracy=0.40, loss=2.74] \n",
-      "Train: 100% 2000/2000 [58:52<00:00,  1.77s/ step, accuracy=0.44, loss=2.50, step=38000] \n",
-      "Valid: 100% 5664/5667 [00:49<00:00, 114.63 uttr/s, accuracy=0.44, loss=2.52] \n",
-      "Train: 100% 2000/2000 [03:15<00:00, 10.23 step/s, accuracy=0.47, loss=2.14, step=4e+4]  \n",
-      "Valid: 100% 5664/5667 [00:49<00:00, 113.74 uttr/s, accuracy=0.46, loss=2.42] \n",
-      "Train:   0% 3/2000 [00:00<02:18, 14.40 step/s, accuracy=0.50, loss=2.19, step=4e+4]"
+      "Train: 100% 2000/2000 [04:21<00:00,  7.66 step/s, accuracy=0.81, loss=1.04, step=42000]  \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 101.18 uttr/s, accuracy=0.63, loss=1.61]\n",
+      "Train: 100% 2000/2000 [04:29<00:00,  7.42 step/s, accuracy=0.81, loss=0.85, step=44000]  \n",
+      "Valid: 100% 5664/5667 [00:56<00:00, 99.96 uttr/s, accuracy=0.65, loss=1.51] \n",
+      "Train: 100% 2000/2000 [04:34<00:00,  7.29 step/s, accuracy=0.50, loss=2.01, step=46000] \n",
+      "Valid: 100% 5664/5667 [00:56<00:00, 100.25 uttr/s, accuracy=0.61, loss=1.70]\n",
+      "Train: 100% 2000/2000 [05:14<00:00,  6.36 step/s, accuracy=0.81, loss=0.64, step=48000] \n",
+      "Valid: 100% 5664/5667 [00:56<00:00, 99.91 uttr/s, accuracy=0.65, loss=1.52] \n",
+      "Train: 100% 2000/2000 [04:24<00:00,  7.55 step/s, accuracy=0.78, loss=0.98, step=5e+4]   \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 101.81 uttr/s, accuracy=0.65, loss=1.51]\n",
+      "Train:   0% 2/2000 [00:00<05:27,  6.11 step/s, accuracy=0.88, loss=0.51, step=5e+4]"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Step 40000, best model saved. (accuracy=0.4620)\n"
+      "Step 50000, best model saved. (accuracy=0.6543)\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Train: 100% 2000/2000 [02:29<00:00, 13.41 step/s, accuracy=0.41, loss=2.54, step=42000]  \n",
-      "Valid: 100% 5664/5667 [00:49<00:00, 114.05 uttr/s, accuracy=0.47, loss=2.38] \n",
-      "Train: 100% 2000/2000 [02:29<00:00, 13.36 step/s, accuracy=0.50, loss=1.75, step=44000]  \n",
-      "Valid: 100% 5664/5667 [00:50<00:00, 111.34 uttr/s, accuracy=0.48, loss=2.34] \n",
-      "Train: 100% 2000/2000 [02:32<00:00, 13.12 step/s, accuracy=0.38, loss=2.68, step=46000] \n",
-      "Valid: 100% 5664/5667 [00:50<00:00, 111.50 uttr/s, accuracy=0.49, loss=2.29] \n",
-      "Train: 100% 2000/2000 [03:17<00:00, 10.11 step/s, accuracy=0.47, loss=2.11, step=48000] \n",
-      "Valid: 100% 5664/5667 [00:49<00:00, 113.97 uttr/s, accuracy=0.49, loss=2.26] \n",
-      "Train: 100% 2000/2000 [02:31<00:00, 13.19 step/s, accuracy=0.50, loss=2.16, step=5e+4]   \n",
-      "Valid: 100% 5664/5667 [00:49<00:00, 113.30 uttr/s, accuracy=0.49, loss=2.27] \n",
-      "Train:   0% 3/2000 [00:00<03:59,  8.35 step/s, accuracy=0.59, loss=1.67, step=5e+4]"
+      "Train: 100% 2000/2000 [04:25<00:00,  7.53 step/s, accuracy=0.78, loss=0.76, step=52000] \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 101.31 uttr/s, accuracy=0.67, loss=1.45]\n",
+      "Train: 100% 2000/2000 [04:22<00:00,  7.63 step/s, accuracy=0.81, loss=0.99, step=54000] \n",
+      "Valid: 100% 5664/5667 [00:56<00:00, 99.69 uttr/s, accuracy=0.66, loss=1.45] \n",
+      "Train: 100% 2000/2000 [05:04<00:00,  6.56 step/s, accuracy=0.75, loss=0.96, step=56000] \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 101.28 uttr/s, accuracy=0.67, loss=1.45]\n",
+      "Train: 100% 2000/2000 [04:25<00:00,  7.54 step/s, accuracy=0.69, loss=1.06, step=58000]  \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 101.55 uttr/s, accuracy=0.66, loss=1.45]\n",
+      "Train: 100% 2000/2000 [04:18<00:00,  7.74 step/s, accuracy=0.72, loss=0.92, step=6e+4]  \n",
+      "Valid: 100% 5664/5667 [01:20<00:00, 70.68 uttr/s, accuracy=0.67, loss=1.43]  \n",
+      "Train:   0% 2/2000 [00:00<05:23,  6.17 step/s, accuracy=0.75, loss=0.91, step=6e+4]"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Step 50000, best model saved. (accuracy=0.4935)\n"
+      "Step 60000, best model saved. (accuracy=0.6713)\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Train: 100% 2000/2000 [02:31<00:00, 13.17 step/s, accuracy=0.66, loss=1.30, step=52000] \n",
-      "Valid: 100% 5664/5667 [00:50<00:00, 111.10 uttr/s, accuracy=0.52, loss=2.17] \n",
-      "Train: 100% 2000/2000 [02:29<00:00, 13.39 step/s, accuracy=0.62, loss=1.75, step=54000] \n",
-      "Valid: 100% 5664/5667 [00:51<00:00, 110.32 uttr/s, accuracy=0.52, loss=2.13] \n",
-      "Train: 100% 2000/2000 [03:19<00:00, 10.05 step/s, accuracy=0.62, loss=1.76, step=56000] \n",
-      "Valid: 100% 5664/5667 [00:49<00:00, 113.57 uttr/s, accuracy=0.51, loss=2.16] \n",
-      "Train: 100% 2000/2000 [02:31<00:00, 13.20 step/s, accuracy=0.59, loss=1.74, step=58000]  \n",
-      "Valid: 100% 5664/5667 [00:50<00:00, 112.20 uttr/s, accuracy=0.52, loss=2.15] \n",
-      "Train: 100% 2000/2000 [02:31<00:00, 13.22 step/s, accuracy=0.53, loss=2.01, step=6e+4]  \n",
-      "Valid: 100% 5664/5667 [00:50<00:00, 111.23 uttr/s, accuracy=0.53, loss=2.11] \n",
-      "Train:   0% 3/2000 [00:00<03:51,  8.64 step/s, accuracy=0.56, loss=1.88, step=6e+4]"
+      "Train: 100% 2000/2000 [1:52:11<00:00,  3.37s/ step, accuracy=0.72, loss=0.88, step=62000]    \n",
+      "Valid: 100% 5664/5667 [05:24<00:00, 17.47 uttr/s, accuracy=0.66, loss=1.48]  \n",
+      "Train: 100% 2000/2000 [03:49<00:00,  8.73 step/s, accuracy=0.78, loss=1.21, step=64000] \n",
+      "Valid: 100% 5664/5667 [00:50<00:00, 112.98 uttr/s, accuracy=0.67, loss=1.46] \n",
+      "Train: 100% 2000/2000 [03:01<00:00, 11.00 step/s, accuracy=0.84, loss=0.86, step=66000]  \n",
+      "Valid: 100% 5664/5667 [00:50<00:00, 112.34 uttr/s, accuracy=0.68, loss=1.40] \n",
+      "Train: 100% 2000/2000 [03:01<00:00, 11.03 step/s, accuracy=0.72, loss=1.02, step=68000] \n",
+      "Valid: 100% 5664/5667 [00:50<00:00, 111.45 uttr/s, accuracy=0.67, loss=1.43] \n",
+      "Train: 100% 2000/2000 [03:00<00:00, 11.06 step/s, accuracy=0.72, loss=1.39, step=7e+4]  \n",
+      "Valid: 100% 5664/5667 [00:51<00:00, 110.38 uttr/s, accuracy=0.69, loss=1.38] \n",
+      "Train:   0% 2/2000 [00:00<04:41,  7.11 step/s, accuracy=0.84, loss=0.73, step=7e+4]"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Step 60000, best model saved. (accuracy=0.5325)\n"
+      "Step 70000, best model saved. (accuracy=0.6896)\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Train: 100% 2000/2000 [02:30<00:00, 13.31 step/s, accuracy=0.69, loss=1.59, step=62000] \n",
-      "Valid: 100% 5664/5667 [00:51<00:00, 110.88 uttr/s, accuracy=0.53, loss=2.12] \n",
-      "Train: 100% 2000/2000 [03:16<00:00, 10.19 step/s, accuracy=0.66, loss=1.62, step=64000] \n",
-      "Valid: 100% 5664/5667 [00:49<00:00, 113.64 uttr/s, accuracy=0.53, loss=2.11] \n",
-      "Train: 100% 2000/2000 [02:30<00:00, 13.33 step/s, accuracy=0.59, loss=1.66, step=66000]  \n",
-      "Valid: 100% 5664/5667 [00:50<00:00, 111.92 uttr/s, accuracy=0.53, loss=2.09] \n",
-      "Train: 100% 2000/2000 [02:29<00:00, 13.34 step/s, accuracy=0.62, loss=1.99, step=68000] \n",
-      "Valid: 100% 5664/5667 [00:51<00:00, 110.77 uttr/s, accuracy=0.53, loss=2.10] \n",
-      "Train: 100% 2000/2000 [02:30<00:00, 13.29 step/s, accuracy=0.47, loss=2.40, step=7e+4]  \n",
-      "Valid: 100% 5664/5667 [00:51<00:00, 110.05 uttr/s, accuracy=0.53, loss=2.09] \n",
+      "Train: 100% 2000/2000 [03:47<00:00,  8.80 step/s, accuracy=0.69, loss=1.05, step=72000] \n",
+      "Valid: 100% 5664/5667 [00:50<00:00, 112.48 uttr/s, accuracy=0.68, loss=1.38] \n",
+      "Train: 100% 2000/2000 [03:01<00:00, 11.05 step/s, accuracy=0.72, loss=1.14, step=74000]  \n",
+      "Valid: 100% 5664/5667 [00:51<00:00, 110.86 uttr/s, accuracy=0.69, loss=1.37] \n",
+      "Train: 100% 2000/2000 [03:01<00:00, 11.04 step/s, accuracy=0.75, loss=0.93, step=76000] \n",
+      "Valid: 100% 5664/5667 [00:51<00:00, 109.91 uttr/s, accuracy=0.69, loss=1.33] \n",
+      "Train: 100% 2000/2000 [03:02<00:00, 10.97 step/s, accuracy=0.81, loss=0.64, step=78000] \n",
+      "Valid: 100% 5664/5667 [00:51<00:00, 109.28 uttr/s, accuracy=0.68, loss=1.39] \n",
+      "Train: 100% 2000/2000 [03:49<00:00,  8.72 step/s, accuracy=0.75, loss=1.18, step=8e+4] \n",
+      "Valid: 100% 5664/5667 [00:50<00:00, 111.48 uttr/s, accuracy=0.68, loss=1.37] \n",
+      "Train:   0% 2/2000 [00:00<04:14,  7.84 step/s, accuracy=0.78, loss=0.73, step=8e+4]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Step 80000, best model saved. (accuracy=0.6916)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Train: 100% 2000/2000 [03:03<00:00, 10.91 step/s, accuracy=0.75, loss=0.94, step=82000]  \n",
+      "Valid: 100% 5664/5667 [00:51<00:00, 110.45 uttr/s, accuracy=0.69, loss=1.36] \n",
+      "Train: 100% 2000/2000 [03:03<00:00, 10.92 step/s, accuracy=0.84, loss=0.58, step=84000] \n",
+      "Valid: 100% 5664/5667 [00:52<00:00, 108.73 uttr/s, accuracy=0.69, loss=1.31] \n",
+      "Train: 100% 2000/2000 [03:03<00:00, 10.90 step/s, accuracy=0.88, loss=0.64, step=86000] \n",
+      "Valid: 100% 5664/5667 [00:52<00:00, 108.61 uttr/s, accuracy=0.71, loss=1.30] \n",
+      "Train: 100% 2000/2000 [03:48<00:00,  8.74 step/s, accuracy=0.88, loss=0.77, step=88000]\n",
+      "Valid: 100% 5664/5667 [00:50<00:00, 111.64 uttr/s, accuracy=0.70, loss=1.32] \n",
+      "Train: 100% 2000/2000 [03:01<00:00, 11.03 step/s, accuracy=0.84, loss=0.75, step=9e+4]   \n",
+      "Valid: 100% 5664/5667 [00:51<00:00, 110.71 uttr/s, accuracy=0.71, loss=1.29] \n",
+      "Train:   0% 2/2000 [00:00<05:07,  6.51 step/s, accuracy=0.88, loss=0.69, step=9e+4]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Step 90000, best model saved. (accuracy=0.7145)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Train: 100% 2000/2000 [03:01<00:00, 11.02 step/s, accuracy=0.94, loss=0.60, step=92000] \n",
+      "Valid: 100% 5664/5667 [00:51<00:00, 109.85 uttr/s, accuracy=0.71, loss=1.26] \n",
+      "Train: 100% 2000/2000 [03:47<00:00,  8.79 step/s, accuracy=0.66, loss=1.01, step=94000] \n",
+      "Valid: 100% 5664/5667 [00:50<00:00, 112.85 uttr/s, accuracy=0.71, loss=1.25] \n",
+      "Train: 100% 2000/2000 [03:01<00:00, 11.02 step/s, accuracy=0.81, loss=0.70, step=96000]\n",
+      "Valid: 100% 5664/5667 [00:50<00:00, 111.18 uttr/s, accuracy=0.71, loss=1.28] \n",
+      "Train: 100% 2000/2000 [03:01<00:00, 11.02 step/s, accuracy=0.88, loss=0.97, step=98000]  \n",
+      "Valid: 100% 5664/5667 [00:51<00:00, 110.06 uttr/s, accuracy=0.72, loss=1.22] \n",
+      "Train: 100% 2000/2000 [03:01<00:00, 11.01 step/s, accuracy=0.84, loss=0.66, step=1e+5]  \n",
+      "Valid: 100% 5664/5667 [00:51<00:00, 109.19 uttr/s, accuracy=0.71, loss=1.26] \n",
+      "Train:   0% 2/2000 [00:00<04:50,  6.88 step/s, accuracy=0.78, loss=0.66, step=1e+5]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Step 100000, best model saved. (accuracy=0.7156)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Train: 100% 2000/2000 [03:47<00:00,  8.79 step/s, accuracy=0.81, loss=0.59, step=102000]\n",
+      "Valid: 100% 5664/5667 [00:50<00:00, 112.58 uttr/s, accuracy=0.72, loss=1.22] \n",
+      "Train: 100% 2000/2000 [03:01<00:00, 11.00 step/s, accuracy=0.84, loss=0.56, step=104000]  \n",
+      "Valid: 100% 5664/5667 [00:50<00:00, 111.12 uttr/s, accuracy=0.72, loss=1.23] \n",
+      "Train: 100% 2000/2000 [03:01<00:00, 11.03 step/s, accuracy=0.75, loss=1.00, step=106000]  \n",
+      "Valid: 100% 5664/5667 [00:51<00:00, 109.72 uttr/s, accuracy=0.72, loss=1.21] \n",
+      "Train: 100% 2000/2000 [03:01<00:00, 11.01 step/s, accuracy=0.75, loss=0.71, step=108000] \n",
+      "Valid: 100% 5664/5667 [00:51<00:00, 109.17 uttr/s, accuracy=0.72, loss=1.22] \n",
+      "Train: 100% 2000/2000 [03:47<00:00,  8.78 step/s, accuracy=0.91, loss=0.63, step=110000] \n",
+      "Valid: 100% 5664/5667 [00:50<00:00, 112.53 uttr/s, accuracy=0.72, loss=1.22] \n",
+      "Train:   0% 2/2000 [00:00<04:43,  7.06 step/s, accuracy=0.81, loss=0.73, step=110002]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Step 110000, best model saved. (accuracy=0.7209)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Train: 100% 2000/2000 [03:01<00:00, 10.99 step/s, accuracy=0.84, loss=0.69, step=112000]\n",
+      "Valid: 100% 5664/5667 [00:50<00:00, 111.08 uttr/s, accuracy=0.72, loss=1.20] \n",
+      "Train: 100% 2000/2000 [03:01<00:00, 11.02 step/s, accuracy=0.81, loss=0.83, step=114000]  \n",
+      "Valid: 100% 5664/5667 [00:51<00:00, 109.81 uttr/s, accuracy=0.73, loss=1.18] \n",
+      "Train: 100% 2000/2000 [03:01<00:00, 11.01 step/s, accuracy=0.72, loss=1.14, step=116000] \n",
+      "Valid: 100% 5664/5667 [00:52<00:00, 108.72 uttr/s, accuracy=0.72, loss=1.18] \n",
+      "Train: 100% 2000/2000 [03:47<00:00,  8.78 step/s, accuracy=0.84, loss=0.72, step=118000] \n",
+      "Valid: 100% 5664/5667 [00:50<00:00, 112.26 uttr/s, accuracy=0.74, loss=1.14] \n",
+      "Train: 100% 2000/2000 [03:01<00:00, 11.02 step/s, accuracy=0.88, loss=0.58, step=120000]  \n",
+      "Valid: 100% 5664/5667 [00:51<00:00, 110.30 uttr/s, accuracy=0.74, loss=1.15] \n",
+      "Train:   0% 2/2000 [00:00<04:53,  6.80 step/s, accuracy=0.88, loss=0.70, step=120002]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Step 120000, best model saved. (accuracy=0.7387)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Train: 100% 2000/2000 [03:01<00:00, 11.02 step/s, accuracy=0.81, loss=0.79, step=122000]  \n",
+      "Valid: 100% 5664/5667 [00:51<00:00, 109.55 uttr/s, accuracy=0.73, loss=1.14] \n",
+      "Train: 100% 2000/2000 [03:02<00:00, 10.97 step/s, accuracy=0.91, loss=0.81, step=124000] \n",
+      "Valid: 100% 5664/5667 [00:52<00:00, 108.22 uttr/s, accuracy=0.74, loss=1.14] \n",
+      "Train: 100% 2000/2000 [03:47<00:00,  8.78 step/s, accuracy=0.81, loss=0.64, step=126000] \n",
+      "Valid: 100% 5664/5667 [00:50<00:00, 112.00 uttr/s, accuracy=0.73, loss=1.15] \n",
+      "Train: 100% 2000/2000 [03:01<00:00, 11.01 step/s, accuracy=0.88, loss=0.50, step=128000]  \n",
+      "Valid: 100% 5664/5667 [00:51<00:00, 110.37 uttr/s, accuracy=0.74, loss=1.13] \n",
+      "Train: 100% 2000/2000 [03:01<00:00, 11.02 step/s, accuracy=0.91, loss=0.35, step=130000]  \n",
+      "Valid: 100% 5664/5667 [00:51<00:00, 109.04 uttr/s, accuracy=0.74, loss=1.11] \n",
+      "Train:   0% 2/2000 [00:00<05:04,  6.57 step/s, accuracy=0.91, loss=0.70, step=130002]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Step 130000, best model saved. (accuracy=0.7449)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Train: 100% 2000/2000 [03:01<00:00, 11.00 step/s, accuracy=0.94, loss=0.47, step=132000] \n",
+      "Valid: 100% 5664/5667 [00:52<00:00, 108.35 uttr/s, accuracy=0.75, loss=1.08] \n",
+      "Train: 100% 2000/2000 [03:47<00:00,  8.79 step/s, accuracy=0.88, loss=0.54, step=134000] \n",
+      "Valid: 100% 5664/5667 [00:50<00:00, 111.59 uttr/s, accuracy=0.75, loss=1.10] \n",
+      "Train: 100% 2000/2000 [03:06<00:00, 10.74 step/s, accuracy=0.81, loss=0.57, step=136000]  \n",
+      "Valid: 100% 5664/5667 [00:51<00:00, 110.18 uttr/s, accuracy=0.75, loss=1.12] \n",
+      "Train: 100% 2000/2000 [03:02<00:00, 10.96 step/s, accuracy=0.91, loss=0.52, step=138000]  \n",
+      "Valid: 100% 5664/5667 [00:51<00:00, 109.58 uttr/s, accuracy=0.75, loss=1.08] \n",
+      "Train: 100% 2000/2000 [03:02<00:00, 10.94 step/s, accuracy=0.88, loss=0.62, step=140000] \n",
+      "Valid: 100% 5664/5667 [00:52<00:00, 108.00 uttr/s, accuracy=0.76, loss=1.08] \n",
+      "Train:   0% 2/2000 [00:00<04:19,  7.69 step/s, accuracy=0.91, loss=0.48, step=140002]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Step 140000, best model saved. (accuracy=0.7569)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Train: 100% 2000/2000 [03:49<00:00,  8.71 step/s, accuracy=0.81, loss=0.77, step=142000] \n",
+      "Valid: 100% 5664/5667 [00:51<00:00, 110.42 uttr/s, accuracy=0.76, loss=1.08] \n",
+      "Train: 100% 2000/2000 [03:11<00:00, 10.42 step/s, accuracy=0.84, loss=0.71, step=144000]  \n",
+      "Valid: 100% 5664/5667 [00:52<00:00, 108.39 uttr/s, accuracy=0.76, loss=1.05] \n",
+      "Train: 100% 2000/2000 [03:12<00:00, 10.39 step/s, accuracy=0.94, loss=0.34, step=146000] \n",
+      "Valid: 100% 5664/5667 [00:54<00:00, 104.59 uttr/s, accuracy=0.75, loss=1.06] \n",
+      "Train: 100% 2000/2000 [03:07<00:00, 10.64 step/s, accuracy=0.88, loss=0.42, step=148000] \n",
+      "Valid: 100% 5664/5667 [00:52<00:00, 107.56 uttr/s, accuracy=0.75, loss=1.07] \n",
+      "Train: 100% 2000/2000 [03:50<00:00,  8.68 step/s, accuracy=0.88, loss=0.58, step=150000] \n",
+      "Valid: 100% 5664/5667 [00:51<00:00, 110.74 uttr/s, accuracy=0.76, loss=1.06] \n",
       "Train:   0% 0/2000 [00:00<?, ? step/s]"
      ]
     },
@@ -638,26 +834,127 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Step 70000, best model saved. (accuracy=0.5344)\n"
+      "Step 150000, best model saved. (accuracy=0.7620)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Train: 100% 2000/2000 [03:07<00:00, 10.69 step/s, accuracy=0.88, loss=0.50, step=152000]  \n",
+      "Valid: 100% 5664/5667 [00:52<00:00, 107.76 uttr/s, accuracy=0.76, loss=1.04] \n",
+      "Train: 100% 2000/2000 [03:03<00:00, 10.88 step/s, accuracy=1.00, loss=0.28, step=154000] \n",
+      "Valid: 100% 5664/5667 [00:52<00:00, 108.88 uttr/s, accuracy=0.76, loss=1.05] \n",
+      "Train: 100% 2000/2000 [03:03<00:00, 10.91 step/s, accuracy=0.94, loss=0.35, step=156000] \n",
+      "Valid: 100% 5664/5667 [00:52<00:00, 108.45 uttr/s, accuracy=0.77, loss=1.02] \n",
+      "Train: 100% 2000/2000 [03:49<00:00,  8.71 step/s, accuracy=0.88, loss=0.55, step=158000] \n",
+      "Valid: 100% 5664/5667 [00:50<00:00, 111.78 uttr/s, accuracy=0.77, loss=1.02] \n",
+      "Train: 100% 2000/2000 [03:02<00:00, 10.95 step/s, accuracy=0.91, loss=0.37, step=160000]  \n",
+      "Valid: 100% 5664/5667 [00:51<00:00, 110.12 uttr/s, accuracy=0.76, loss=1.05] \n",
+      "Train:   0% 2/2000 [00:00<04:30,  7.39 step/s, accuracy=0.91, loss=0.40, step=160002]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Step 160000, best model saved. (accuracy=0.7675)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Train: 100% 2000/2000 [03:02<00:00, 10.94 step/s, accuracy=0.84, loss=0.78, step=162000] \n",
+      "Valid: 100% 5664/5667 [00:51<00:00, 109.17 uttr/s, accuracy=0.76, loss=1.04] \n",
+      "Train: 100% 2000/2000 [03:04<00:00, 10.86 step/s, accuracy=0.88, loss=0.39, step=164000] \n",
+      "Valid: 100% 5664/5667 [00:52<00:00, 108.03 uttr/s, accuracy=0.77, loss=1.04] \n",
+      "Train: 100% 2000/2000 [03:52<00:00,  8.60 step/s, accuracy=0.84, loss=0.61, step=166000]\n",
+      "Valid: 100% 5664/5667 [00:51<00:00, 109.95 uttr/s, accuracy=0.77, loss=1.01] \n",
+      "Train: 100% 2000/2000 [03:06<00:00, 10.70 step/s, accuracy=0.88, loss=0.82, step=168000]  \n",
+      "Valid: 100% 5664/5667 [00:52<00:00, 108.83 uttr/s, accuracy=0.77, loss=1.02] \n",
+      "Train: 100% 2000/2000 [03:06<00:00, 10.73 step/s, accuracy=0.88, loss=0.35, step=170000] \n",
+      "Valid: 100% 5664/5667 [00:53<00:00, 105.75 uttr/s, accuracy=0.77, loss=1.01] \n",
+      "Train:   0% 1/2000 [00:00<05:57,  5.60 step/s, accuracy=0.94, loss=0.34, step=170001]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Step 170000, best model saved. (accuracy=0.7691)\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "\n"
+      "Train: 100% 2000/2000 [03:09<00:00, 10.58 step/s, accuracy=0.97, loss=0.38, step=172000] \n",
+      "Valid: 100% 5664/5667 [00:52<00:00, 106.87 uttr/s, accuracy=0.77, loss=1.01] \n",
+      "Train: 100% 2000/2000 [03:51<00:00,  8.63 step/s, accuracy=0.84, loss=0.58, step=174000]\n",
+      "Valid: 100% 5664/5667 [00:52<00:00, 107.02 uttr/s, accuracy=0.77, loss=1.00] \n",
+      "Train: 100% 2000/2000 [03:09<00:00, 10.55 step/s, accuracy=0.91, loss=0.35, step=176000]  \n",
+      "Valid: 100% 5664/5667 [00:52<00:00, 108.08 uttr/s, accuracy=0.77, loss=0.99] \n",
+      "Train: 100% 2000/2000 [03:07<00:00, 10.66 step/s, accuracy=0.88, loss=0.37, step=178000] \n",
+      "Valid: 100% 5664/5667 [00:52<00:00, 107.25 uttr/s, accuracy=0.77, loss=1.03] \n",
+      "Train: 100% 2000/2000 [03:11<00:00, 10.46 step/s, accuracy=0.91, loss=0.27, step=180000] \n",
+      "Valid: 100% 5664/5667 [00:52<00:00, 107.19 uttr/s, accuracy=0.77, loss=1.02] \n",
+      "Train:   0% 2/2000 [00:00<04:58,  6.69 step/s, accuracy=0.88, loss=0.46, step=180002]"
      ]
     },
     {
-     "ename": "NameError",
-     "evalue": "name 'train_end_log' is not defined",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
-      "Cell \u001b[0;32mIn[10], line 103\u001b[0m\n\u001b[1;32m    100\u001b[0m     train_end_log()\n\u001b[1;32m    102\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;18m__name__\u001b[39m \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m__main__\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m--> 103\u001b[0m     \u001b[43mmain\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mparse_args\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n",
-      "Cell \u001b[0;32mIn[10], line 100\u001b[0m, in \u001b[0;36mmain\u001b[0;34m(data_dir, save_path, batch_size, n_workers, valid_steps, warmup_steps, total_steps, save_steps)\u001b[0m\n\u001b[1;32m     97\u001b[0m         pbar\u001b[38;5;241m.\u001b[39mwrite(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mStep \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mstep\u001b[38;5;250m \u001b[39m\u001b[38;5;241m+\u001b[39m\u001b[38;5;250m \u001b[39m\u001b[38;5;241m1\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, best model saved. (accuracy=\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mbest_accuracy\u001b[38;5;132;01m:\u001b[39;00m\u001b[38;5;124m.4f\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m)\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m     99\u001b[0m pbar\u001b[38;5;241m.\u001b[39mclose()\n\u001b[0;32m--> 100\u001b[0m \u001b[43mtrain_end_log\u001b[49m()\n",
-      "\u001b[0;31mNameError\u001b[0m: name 'train_end_log' is not defined"
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Step 180000, best model saved. (accuracy=0.7703)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Train: 100% 2000/2000 [03:59<00:00,  8.35 step/s, accuracy=0.81, loss=0.59, step=182000]\n",
+      "Valid: 100% 5664/5667 [00:52<00:00, 107.89 uttr/s, accuracy=0.77, loss=0.99] \n",
+      "Train: 100% 2000/2000 [03:34<00:00,  9.32 step/s, accuracy=0.84, loss=0.55, step=184000]  \n",
+      "Valid: 100% 5664/5667 [00:53<00:00, 105.38 uttr/s, accuracy=0.78, loss=0.98] \n",
+      "Train: 100% 2000/2000 [03:37<00:00,  9.21 step/s, accuracy=0.97, loss=0.24, step=186000] \n",
+      "Valid: 100% 5664/5667 [00:56<00:00, 100.63 uttr/s, accuracy=0.77, loss=0.98] \n",
+      "Train: 100% 2000/2000 [04:01<00:00,  8.30 step/s, accuracy=0.88, loss=0.42, step=188000] \n",
+      "Valid: 100% 5664/5667 [00:50<00:00, 111.92 uttr/s, accuracy=0.77, loss=1.00] \n",
+      "Train: 100% 2000/2000 [03:06<00:00, 10.73 step/s, accuracy=0.97, loss=0.28, step=190000]\n",
+      "Valid: 100% 5664/5667 [00:51<00:00, 109.99 uttr/s, accuracy=0.77, loss=1.00] \n",
+      "Train:   0% 2/2000 [00:00<04:56,  6.75 step/s, accuracy=0.97, loss=0.43, step=190002]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Step 190000, best model saved. (accuracy=0.7751)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Train: 100% 2000/2000 [03:04<00:00, 10.82 step/s, accuracy=0.88, loss=0.61, step=192000]  \n",
+      "Valid: 100% 5664/5667 [00:53<00:00, 106.29 uttr/s, accuracy=0.78, loss=0.99] \n",
+      "Train: 100% 2000/2000 [03:08<00:00, 10.60 step/s, accuracy=0.94, loss=0.23, step=194000] \n",
+      "Valid: 100% 5664/5667 [00:52<00:00, 107.65 uttr/s, accuracy=0.77, loss=1.00] \n",
+      "Train: 100% 2000/2000 [03:51<00:00,  8.63 step/s, accuracy=0.91, loss=0.30, step=196000] \n",
+      "Valid: 100% 5664/5667 [00:50<00:00, 111.25 uttr/s, accuracy=0.77, loss=0.99] \n",
+      "Train: 100% 2000/2000 [03:08<00:00, 10.59 step/s, accuracy=0.94, loss=0.46, step=198000]\n",
+      "Valid: 100% 5664/5667 [00:51<00:00, 109.03 uttr/s, accuracy=0.77, loss=1.00] \n",
+      "Train: 100% 2000/2000 [03:13<00:00, 10.33 step/s, accuracy=0.91, loss=0.36, step=2e+5]    \n",
+      "Valid: 100% 5664/5667 [00:53<00:00, 105.60 uttr/s, accuracy=0.77, loss=1.03] \n",
+      "Train:   0% 0/2000 [00:00<?, ? step/s]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Step 200000, best model saved. (accuracy=0.7779)\n"
      ]
     }
    ],
@@ -680,7 +977,7 @@
     "        \"valid_steps\": 2000,\n",
     "        \"warmup_steps\": 1000,\n",
     "        \"save_steps\": 10000,\n",
-    "        \"total_steps\": 70000,\n",
+    "        \"total_steps\": 200000,\n",
     "    }\n",
     "\n",
     "    return config\n",
@@ -698,7 +995,7 @@
     "):\n",
     "    \"\"\"Main function.\"\"\"\n",
     "    device = \"cuda\" if torch.cuda.is_available() else 'mps' if torch.backends.mps.is_available() else 'cpu'\n",
-    "\n",
+    "    #device = 'cpu'\n",
     "    print(f\"[Info]: Use {device} now!\")\n",
     "\n",
     "    train_loader, valid_loader, speaker_num = get_dataloader(data_dir, batch_size, n_workers)\n",
@@ -706,6 +1003,7 @@
     "    print(f\"[Info]: Finish loading data!\",flush = True)\n",
     "\n",
     "    model = Classifier(n_spks=speaker_num).to(device)\n",
+    "    model.load_state_dict(torch.load('./model.ckpt'))\n",
     "    criterion = nn.CrossEntropyLoss()\n",
     "    optimizer = AdamW(model.parameters(), lr=1e-3)\n",
     "    scheduler = get_cosine_schedule_with_warmup(optimizer, warmup_steps, total_steps)\n",
@@ -780,7 +1078,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 8,
    "metadata": {
     "id": "efS4pCmAJXJH"
    },
@@ -807,7 +1105,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 9,
    "metadata": {
     "colab": {
      "base_uri": "https://localhost:8080/",
@@ -842,7 +1140,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "5d35d5a6d5e54cfc8d64535013b8e7dd",
+       "model_id": "5026b3ed28e049fe9115b2ec71dcbe12",
        "version_major": 2,
        "version_minor": 0
       },