20230330 step 160_000 acc 0.664901, will contiue running

ML2023_hw04.ipynb

@@ -68,7 +68,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 3,
    "metadata": {
     "id": "E6burzCXIyuA"
    },
@@ -131,7 +131,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 4,
    "metadata": {
     "id": "KpuGxl4CI2pr"
    },
@@ -160,7 +160,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 5,
    "metadata": {
     "id": "B7c2gZYoJDRS"
    },
@@ -236,7 +236,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 6,
    "metadata": {
     "id": "iXZ5B0EKJGs8"
    },
@@ -248,7 +248,7 @@
     "import torchaudio\n",
     "\n",
     "class Classifier(nn.Module):\n",
-    "    def __init__(self, d_model=160, n_spks=600, dropout=0.5):\n",
+    "    def __init__(self, d_model=160, n_spks=600, dropout=0.6):\n",
     "        super().__init__()\n",
     "        # Project the dimension of features from that of input into d_model.\n",
     "        self.prenet = nn.Linear(40, d_model)\n",
@@ -282,6 +282,7 @@
     "        #out = out.permute(1, 0, 2)\n",
     "        # The encoder layer expect features in the shape of (length, batch size, d_model).\n",
     "        device = \"cuda\" if torch.cuda.is_available() else 'mps' if torch.backends.mps.is_available() else 'cpu'\n",
+    "        #device = \"cpu\"\n",
     "        #lengths = torch.randint(out.size(1), (int(out.size(0)),)).to(device)\n",
     "        #lengths[torch.argmax(lengths)] = out.size(1)\n",
     "        lengths = torch.full((int(out.size(0)),), out.size(1)).to(device)\n",
@@ -460,6 +461,58 @@
     "    return running_accuracy / len(dataloader)"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Additive-Margin-Softmax\n",
+    "Reference: https://github.com/Leethony/Additive-Margin-Softmax-Loss-Pytorch"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import torch\n",
+    "import torch.nn as nn\n",
+    "import torch.nn.functional as F\n",
+    "\n",
+    "class AdMSoftmaxLoss(nn.Module):\n",
+    "\n",
+    "    def __init__(self, in_features, out_features, s=30.0, m=0.4):\n",
+    "        '''\n",
+    "        AM Softmax Loss\n",
+    "        '''\n",
+    "        super(AdMSoftmaxLoss, self).__init__()\n",
+    "        self.s = s\n",
+    "        self.m = m\n",
+    "        self.in_features = in_features\n",
+    "        self.out_features = out_features\n",
+    "        #self.fc = nn.Linear(in_features, out_features, bias=False)\n",
+    "\n",
+    "    def forward(self, x, labels):\n",
+    "        '''\n",
+    "        input shape (N, in_features)\n",
+    "        '''\n",
+    "        assert len(x) == len(labels)\n",
+    "        assert torch.min(labels) >= 0\n",
+    "        assert torch.max(labels) < self.out_features\n",
+    "        \n",
+    "        #for W in self.fc.parameters():\n",
+    "        #    W = F.normalize(W, dim=1)\n",
+    "\n",
+    "        x = F.normalize(x, dim=1)\n",
+    "        #x = x.view(-1, x.size(0))\n",
+    "        wf = x\n",
+    "        numerator = self.s * (torch.diagonal(wf.transpose(0, 1)[labels]) - self.m)\n",
+    "        excl = torch.cat([torch.cat((wf[i, :y], wf[i, y+1:])).unsqueeze(0) for i, y in enumerate(labels)], dim=0)\n",
+    "        denominator = torch.exp(numerator) + torch.sum(torch.exp(self.s * excl), dim=1)\n",
+    "        L = numerator - torch.log(denominator)\n",
+    "        return -torch.mean(L)"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {
@@ -471,7 +524,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 11,
    "metadata": {
     "colab": {
      "base_uri": "https://localhost:8080/"
@@ -493,16 +546,16 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Train: 100% 2000/2000 [04:31<00:00,  7.36 step/s, accuracy=0.22, loss=3.40, step=2000]  \n",
-      "Valid: 100% 5664/5667 [00:53<00:00, 106.62 uttr/s, accuracy=0.24, loss=3.57]\n",
-      "Train: 100% 2000/2000 [04:38<00:00,  7.17 step/s, accuracy=0.34, loss=2.71, step=4000]  \n",
-      "Valid: 100% 5664/5667 [00:53<00:00, 104.90 uttr/s, accuracy=0.42, loss=2.59]\n",
-      "Train: 100% 2000/2000 [04:33<00:00,  7.31 step/s, accuracy=0.59, loss=1.64, step=6000] \n",
-      "Valid: 100% 5664/5667 [00:54<00:00, 103.97 uttr/s, accuracy=0.55, loss=2.03]\n",
-      "Train: 100% 2000/2000 [12:48<00:00,  2.60 step/s, accuracy=0.56, loss=2.18, step=8000]   \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 108.64 uttr/s, accuracy=0.60, loss=1.74]\n",
-      "Train: 100% 2000/2000 [04:35<00:00,  7.26 step/s, accuracy=0.72, loss=1.23, step=1e+4]  \n",
-      "Valid: 100% 5664/5667 [00:54<00:00, 103.98 uttr/s, accuracy=0.64, loss=1.60]\n",
+      "Train: 100% 2000/2000 [05:20<00:00,  6.25 step/s, accuracy=0.16, loss=15.88, step=2000]  \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 102.81 uttr/s, accuracy=0.24, loss=15.80]\n",
+      "Train: 100% 2000/2000 [05:17<00:00,  6.29 step/s, accuracy=0.31, loss=14.76, step=4000]  \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 101.96 uttr/s, accuracy=0.39, loss=14.58]\n",
+      "Train: 100% 2000/2000 [05:17<00:00,  6.29 step/s, accuracy=0.56, loss=13.00, step=6000] \n",
+      "Valid: 100% 5664/5667 [00:56<00:00, 100.81 uttr/s, accuracy=0.46, loss=13.85]\n",
+      "Train: 100% 2000/2000 [06:03<00:00,  5.51 step/s, accuracy=0.53, loss=13.60, step=8000] \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 102.79 uttr/s, accuracy=0.48, loss=13.37]\n",
+      "Train: 100% 2000/2000 [05:16<00:00,  6.31 step/s, accuracy=0.53, loss=13.56, step=1e+4]  \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 101.78 uttr/s, accuracy=0.51, loss=12.95]\n",
       "Train:   0% 0/2000 [00:00<?, ? step/s]"
      ]
     },
@@ -510,23 +563,23 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Step 10000, best model saved. (accuracy=0.6398)\n"
+      "Step 10000, best model saved. (accuracy=0.5115)\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Train: 100% 2000/2000 [04:35<00:00,  7.26 step/s, accuracy=0.72, loss=1.43, step=12000]  \n",
-      "Valid: 100% 5664/5667 [00:54<00:00, 103.49 uttr/s, accuracy=0.66, loss=1.55]\n",
-      "Train: 100% 2000/2000 [04:37<00:00,  7.22 step/s, accuracy=0.88, loss=0.73, step=14000] \n",
-      "Valid: 100% 5664/5667 [00:53<00:00, 105.48 uttr/s, accuracy=0.70, loss=1.34]\n",
-      "Train: 100% 2000/2000 [05:48<00:00,  5.74 step/s, accuracy=0.72, loss=1.05, step=16000] \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 108.00 uttr/s, accuracy=0.72, loss=1.28]\n",
-      "Train: 100% 2000/2000 [04:54<00:00,  6.79 step/s, accuracy=0.84, loss=0.52, step=18000]  \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 107.34 uttr/s, accuracy=0.73, loss=1.28]\n",
-      "Train: 100% 2000/2000 [05:03<00:00,  6.59 step/s, accuracy=0.81, loss=0.69, step=2e+4]   \n",
-      "Valid: 100% 5664/5667 [00:54<00:00, 103.80 uttr/s, accuracy=0.73, loss=1.25]\n",
+      "Train: 100% 2000/2000 [05:17<00:00,  6.30 step/s, accuracy=0.56, loss=12.15, step=12000]  \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 101.22 uttr/s, accuracy=0.50, loss=12.94]\n",
+      "Train: 100% 2000/2000 [20:55<00:00,  1.59 step/s, accuracy=0.62, loss=12.13, step=14000]   \n",
+      "Valid: 100% 5664/5667 [00:54<00:00, 103.04 uttr/s, accuracy=0.50, loss=12.66]\n",
+      "Train: 100% 2000/2000 [06:20<00:00,  5.26 step/s, accuracy=0.62, loss=12.90, step=16000] \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 102.33 uttr/s, accuracy=0.53, loss=12.25]\n",
+      "Train: 100% 2000/2000 [05:18<00:00,  6.28 step/s, accuracy=0.78, loss=10.41, step=18000]  \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 101.64 uttr/s, accuracy=0.51, loss=12.37]\n",
+      "Train: 100% 2000/2000 [32:05<00:00,  1.04 step/s, accuracy=0.59, loss=11.95, step=2e+4]     \n",
+      "Valid: 100% 5664/5667 [11:14<00:00,  8.39 uttr/s, accuracy=0.52, loss=12.15] \n",
       "Train:   0% 0/2000 [00:00<?, ? step/s]"
      ]
     },
@@ -534,23 +587,23 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Step 20000, best model saved. (accuracy=0.7334)\n"
+      "Step 20000, best model saved. (accuracy=0.5268)\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Train: 100% 2000/2000 [05:20<00:00,  6.24 step/s, accuracy=0.91, loss=0.48, step=22000] \n",
-      "Valid: 100% 5664/5667 [00:56<00:00, 99.70 uttr/s, accuracy=0.74, loss=1.22] \n",
-      "Train: 100% 2000/2000 [05:47<00:00,  5.75 step/s, accuracy=0.78, loss=0.56, step=24000] \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 108.01 uttr/s, accuracy=0.75, loss=1.21]\n",
-      "Train: 100% 2000/2000 [04:58<00:00,  6.69 step/s, accuracy=0.88, loss=0.34, step=26000]  \n",
-      "Valid: 100% 5664/5667 [00:53<00:00, 105.23 uttr/s, accuracy=0.75, loss=1.20]\n",
-      "Train: 100% 2000/2000 [05:29<00:00,  6.06 step/s, accuracy=0.88, loss=0.37, step=28000]  \n",
-      "Valid: 100% 5664/5667 [00:56<00:00, 100.46 uttr/s, accuracy=0.77, loss=1.12]\n",
-      "Train: 100% 2000/2000 [05:06<00:00,  6.52 step/s, accuracy=0.88, loss=0.58, step=3e+4]  \n",
-      "Valid: 100% 5664/5667 [00:55<00:00, 101.50 uttr/s, accuracy=0.78, loss=1.05]\n",
+      "Train: 100% 2000/2000 [05:22<00:00,  6.21 step/s, accuracy=0.59, loss=10.88, step=22000] \n",
+      "Valid: 100% 5664/5667 [00:56<00:00, 101.14 uttr/s, accuracy=0.54, loss=11.92]\n",
+      "Train: 100% 2000/2000 [06:08<00:00,  5.43 step/s, accuracy=0.69, loss=11.18, step=24000] \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 101.20 uttr/s, accuracy=0.53, loss=11.89]\n",
+      "Train: 100% 2000/2000 [05:21<00:00,  6.22 step/s, accuracy=0.69, loss=9.22, step=26000]   \n",
+      "Valid: 100% 5664/5667 [00:56<00:00, 100.41 uttr/s, accuracy=0.53, loss=11.87]\n",
+      "Train: 100% 2000/2000 [05:20<00:00,  6.24 step/s, accuracy=0.66, loss=11.01, step=28000]  \n",
+      "Valid: 100% 5664/5667 [00:56<00:00, 99.48 uttr/s, accuracy=0.53, loss=11.84] \n",
+      "Train: 100% 2000/2000 [05:33<00:00,  5.99 step/s, accuracy=0.72, loss=10.01, step=3e+4]  \n",
+      "Valid: 100% 5664/5667 [00:58<00:00, 96.75 uttr/s, accuracy=0.52, loss=11.88] \n",
       "Train:   0% 0/2000 [00:00<?, ? step/s]"
      ]
     },
@@ -558,23 +611,23 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Step 30000, best model saved. (accuracy=0.7800)\n"
+      "Step 30000, best model saved. (accuracy=0.5395)\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Train: 100% 2000/2000 [05:52<00:00,  5.67 step/s, accuracy=0.94, loss=0.20, step=32000] \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 108.10 uttr/s, accuracy=0.79, loss=1.06]\n",
-      "Train: 100% 2000/2000 [04:51<00:00,  6.87 step/s, accuracy=0.84, loss=0.73, step=34000]  \n",
-      "Valid: 100% 5664/5667 [00:53<00:00, 106.68 uttr/s, accuracy=0.78, loss=1.09]\n",
-      "Train: 100% 2000/2000 [04:43<00:00,  7.05 step/s, accuracy=0.88, loss=0.55, step=36000]  \n",
-      "Valid: 100% 5664/5667 [00:53<00:00, 105.69 uttr/s, accuracy=0.78, loss=1.08]\n",
-      "Train: 100% 2000/2000 [04:47<00:00,  6.97 step/s, accuracy=0.91, loss=0.39, step=38000] \n",
-      "Valid: 100% 5664/5667 [00:55<00:00, 102.95 uttr/s, accuracy=0.79, loss=1.06]\n",
-      "Train: 100% 2000/2000 [05:34<00:00,  5.98 step/s, accuracy=0.91, loss=0.51, step=4e+4]  \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 108.32 uttr/s, accuracy=0.81, loss=0.97]\n",
+      "Train: 100% 2000/2000 [06:24<00:00,  5.20 step/s, accuracy=0.59, loss=10.31, step=32000] \n",
+      "Valid: 100% 5664/5667 [00:56<00:00, 99.93 uttr/s, accuracy=0.52, loss=11.90] \n",
+      "Train: 100% 2000/2000 [05:51<00:00,  5.69 step/s, accuracy=0.59, loss=10.06, step=34000]  \n",
+      "Valid: 100% 5664/5667 [00:58<00:00, 96.96 uttr/s, accuracy=0.52, loss=11.77] \n",
+      "Train: 100% 2000/2000 [06:11<00:00,  5.38 step/s, accuracy=0.47, loss=10.95, step=36000]  \n",
+      "Valid: 100% 5664/5667 [00:58<00:00, 96.82 uttr/s, accuracy=0.54, loss=11.50] \n",
+      "Train: 100% 2000/2000 [05:53<00:00,  5.66 step/s, accuracy=0.56, loss=11.36, step=38000] \n",
+      "Valid: 100% 5664/5667 [00:57<00:00, 98.39 uttr/s, accuracy=0.55, loss=11.38] \n",
+      "Train: 100% 2000/2000 [06:32<00:00,  5.09 step/s, accuracy=0.59, loss=10.49, step=4e+4]  \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 102.03 uttr/s, accuracy=0.55, loss=11.27]\n",
       "Train:   0% 0/2000 [00:00<?, ? step/s]"
      ]
     },
@@ -582,23 +635,23 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Step 40000, best model saved. (accuracy=0.8069)\n"
+      "Step 40000, best model saved. (accuracy=0.5528)\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Train: 100% 2000/2000 [04:44<00:00,  7.04 step/s, accuracy=0.97, loss=0.38, step=42000]  \n",
-      "Valid: 100% 5664/5667 [00:53<00:00, 105.66 uttr/s, accuracy=0.79, loss=1.01]\n",
-      "Train: 100% 2000/2000 [05:02<00:00,  6.61 step/s, accuracy=0.84, loss=0.46, step=44000]  \n",
-      "Valid: 100% 5664/5667 [00:55<00:00, 102.54 uttr/s, accuracy=0.80, loss=1.01]\n",
-      "Train: 100% 2000/2000 [05:07<00:00,  6.51 step/s, accuracy=0.78, loss=0.68, step=46000] \n",
-      "Valid: 100% 5664/5667 [00:57<00:00, 97.74 uttr/s, accuracy=0.80, loss=0.99] \n",
-      "Train: 100% 2000/2000 [05:44<00:00,  5.80 step/s, accuracy=0.94, loss=0.35, step=48000] \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 107.12 uttr/s, accuracy=0.81, loss=0.98]\n",
-      "Train: 100% 2000/2000 [04:51<00:00,  6.85 step/s, accuracy=0.88, loss=0.30, step=5e+4]   \n",
-      "Valid: 100% 5664/5667 [00:53<00:00, 105.84 uttr/s, accuracy=0.81, loss=0.98]\n",
+      "Train: 100% 2000/2000 [05:31<00:00,  6.03 step/s, accuracy=0.75, loss=7.78, step=42000]   \n",
+      "Valid: 100% 5664/5667 [00:56<00:00, 100.28 uttr/s, accuracy=0.56, loss=11.16]\n",
+      "Train: 100% 2000/2000 [05:58<00:00,  5.58 step/s, accuracy=0.56, loss=10.43, step=44000]  \n",
+      "Valid: 100% 5664/5667 [00:58<00:00, 97.24 uttr/s, accuracy=0.56, loss=11.13] \n",
+      "Train: 100% 2000/2000 [05:47<00:00,  5.75 step/s, accuracy=0.66, loss=8.46, step=46000]  \n",
+      "Valid: 100% 5664/5667 [00:57<00:00, 98.94 uttr/s, accuracy=0.56, loss=11.13] \n",
+      "Train: 100% 2000/2000 [06:33<00:00,  5.08 step/s, accuracy=0.78, loss=9.20, step=48000]  \n",
+      "Valid: 100% 5664/5667 [00:57<00:00, 99.22 uttr/s, accuracy=0.56, loss=11.20] \n",
+      "Train: 100% 2000/2000 [05:49<00:00,  5.72 step/s, accuracy=0.78, loss=9.64, step=5e+4]    \n",
+      "Valid: 100% 5664/5667 [00:56<00:00, 100.48 uttr/s, accuracy=0.55, loss=11.18]\n",
       "Train:   0% 0/2000 [00:00<?, ? step/s]"
      ]
     },
@@ -606,23 +659,23 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Step 50000, best model saved. (accuracy=0.8120)\n"
+      "Step 50000, best model saved. (accuracy=0.5627)\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Train: 100% 2000/2000 [05:03<00:00,  6.59 step/s, accuracy=0.81, loss=0.74, step=52000] \n",
-      "Valid: 100% 5664/5667 [00:54<00:00, 104.29 uttr/s, accuracy=0.81, loss=0.93]\n",
-      "Train: 100% 2000/2000 [05:12<00:00,  6.40 step/s, accuracy=0.78, loss=0.70, step=54000] \n",
-      "Valid: 100% 5664/5667 [00:56<00:00, 100.11 uttr/s, accuracy=0.82, loss=0.95]\n",
-      "Train: 100% 2000/2000 [05:59<00:00,  5.56 step/s, accuracy=0.94, loss=0.25, step=56000] \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 107.08 uttr/s, accuracy=0.82, loss=0.94]\n",
-      "Train: 100% 2000/2000 [05:04<00:00,  6.58 step/s, accuracy=0.94, loss=0.16, step=58000]  \n",
-      "Valid: 100% 5664/5667 [00:53<00:00, 106.15 uttr/s, accuracy=0.81, loss=0.93]\n",
-      "Train: 100% 2000/2000 [05:08<00:00,  6.48 step/s, accuracy=0.78, loss=0.60, step=6e+4]  \n",
-      "Valid: 100% 5664/5667 [00:55<00:00, 101.84 uttr/s, accuracy=0.82, loss=0.91]\n",
+      "Train: 100% 2000/2000 [05:20<00:00,  6.25 step/s, accuracy=0.72, loss=8.79, step=52000]  \n",
+      "Valid: 100% 5664/5667 [00:57<00:00, 99.23 uttr/s, accuracy=0.55, loss=11.24] \n",
+      "Train: 100% 2000/2000 [05:29<00:00,  6.06 step/s, accuracy=0.56, loss=10.06, step=54000] \n",
+      "Valid: 100% 5664/5667 [00:57<00:00, 97.98 uttr/s, accuracy=0.57, loss=10.90] \n",
+      "Train: 100% 2000/2000 [06:14<00:00,  5.34 step/s, accuracy=0.69, loss=8.61, step=56000] \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 101.23 uttr/s, accuracy=0.57, loss=10.99]\n",
+      "Train: 100% 2000/2000 [05:20<00:00,  6.25 step/s, accuracy=0.53, loss=11.35, step=58000]  \n",
+      "Valid: 100% 5664/5667 [00:56<00:00, 99.57 uttr/s, accuracy=0.57, loss=10.80] \n",
+      "Train: 100% 2000/2000 [05:39<00:00,  5.89 step/s, accuracy=0.50, loss=9.96, step=6e+4]   \n",
+      "Valid: 100% 5664/5667 [00:57<00:00, 97.99 uttr/s, accuracy=0.56, loss=10.96] \n",
       "Train:   0% 0/2000 [00:00<?, ? step/s]"
      ]
     },
@@ -630,23 +683,23 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Step 60000, best model saved. (accuracy=0.8245)\n"
+      "Step 60000, best model saved. (accuracy=0.5742)\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Train: 100% 2000/2000 [05:16<00:00,  6.33 step/s, accuracy=0.91, loss=0.34, step=62000] \n",
-      "Valid: 100% 5664/5667 [00:55<00:00, 102.56 uttr/s, accuracy=0.83, loss=0.88]\n",
-      "Train: 100% 2000/2000 [18:59<00:00,  1.76 step/s, accuracy=0.97, loss=0.21, step=64000]   \n",
-      "Valid: 100% 5664/5667 [11:08<00:00,  8.48 uttr/s, accuracy=0.84, loss=0.87] \n",
-      "Train: 100% 2000/2000 [10:41<00:00,  3.12 step/s, accuracy=0.84, loss=0.54, step=66000]    \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 108.57 uttr/s, accuracy=0.83, loss=0.87]\n",
-      "Train: 100% 2000/2000 [04:28<00:00,  7.46 step/s, accuracy=0.91, loss=0.22, step=68000] \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 107.19 uttr/s, accuracy=0.83, loss=0.87]\n",
-      "Train: 100% 2000/2000 [04:25<00:00,  7.53 step/s, accuracy=0.97, loss=0.15, step=7e+4]  \n",
-      "Valid: 100% 5664/5667 [00:53<00:00, 106.26 uttr/s, accuracy=0.83, loss=0.89]\n",
+      "Train: 100% 2000/2000 [05:42<00:00,  5.84 step/s, accuracy=0.59, loss=10.50, step=62000] \n",
+      "Valid: 100% 5664/5667 [00:58<00:00, 96.09 uttr/s, accuracy=0.58, loss=10.73] \n",
+      "Train: 100% 2000/2000 [06:45<00:00,  4.93 step/s, accuracy=0.84, loss=7.73, step=64000]  \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 102.12 uttr/s, accuracy=0.57, loss=10.81]\n",
+      "Train: 100% 2000/2000 [05:31<00:00,  6.03 step/s, accuracy=0.66, loss=9.90, step=66000]   \n",
+      "Valid: 100% 5664/5667 [00:56<00:00, 100.17 uttr/s, accuracy=0.58, loss=10.76]\n",
+      "Train: 100% 2000/2000 [05:50<00:00,  5.71 step/s, accuracy=0.88, loss=6.91, step=68000] \n",
+      "Valid: 100% 5664/5667 [00:59<00:00, 94.84 uttr/s, accuracy=0.59, loss=10.68] \n",
+      "Train: 100% 2000/2000 [05:59<00:00,  5.57 step/s, accuracy=0.81, loss=7.51, step=7e+4]  \n",
+      "Valid: 100% 5664/5667 [00:58<00:00, 97.50 uttr/s, accuracy=0.59, loss=10.65] \n",
       "Train:   0% 0/2000 [00:00<?, ? step/s]"
      ]
     },
@@ -654,23 +707,23 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Step 70000, best model saved. (accuracy=0.8353)\n"
+      "Step 70000, best model saved. (accuracy=0.5855)\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Train: 100% 2000/2000 [10:44<00:00,  3.10 step/s, accuracy=0.97, loss=0.14, step=72000]  \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 107.85 uttr/s, accuracy=0.83, loss=0.90]\n",
-      "Train: 100% 2000/2000 [04:31<00:00,  7.37 step/s, accuracy=0.94, loss=0.22, step=74000]  \n",
-      "Valid: 100% 5664/5667 [00:53<00:00, 106.09 uttr/s, accuracy=0.83, loss=0.91]\n",
-      "Train: 100% 2000/2000 [04:31<00:00,  7.38 step/s, accuracy=0.88, loss=0.51, step=76000] \n",
-      "Valid: 100% 5664/5667 [00:55<00:00, 101.38 uttr/s, accuracy=0.83, loss=0.86]\n",
-      "Train: 100% 2000/2000 [04:52<00:00,  6.83 step/s, accuracy=0.84, loss=0.30, step=78000] \n",
-      "Valid: 100% 5664/5667 [00:54<00:00, 103.45 uttr/s, accuracy=0.84, loss=0.86]\n",
-      "Train: 100% 2000/2000 [05:30<00:00,  6.05 step/s, accuracy=0.91, loss=0.33, step=8e+4]   \n",
-      "Valid: 100% 5664/5667 [00:54<00:00, 104.53 uttr/s, accuracy=0.85, loss=0.86]\n",
+      "Train: 100% 2000/2000 [06:24<00:00,  5.20 step/s, accuracy=0.62, loss=10.97, step=72000] \n",
+      "Valid: 100% 5664/5667 [00:56<00:00, 100.47 uttr/s, accuracy=0.59, loss=10.49]\n",
+      "Train: 100% 2000/2000 [05:24<00:00,  6.16 step/s, accuracy=0.72, loss=8.54, step=74000]  \n",
+      "Valid: 100% 5664/5667 [00:56<00:00, 99.56 uttr/s, accuracy=0.60, loss=10.50] \n",
+      "Train: 100% 2000/2000 [28:45<00:00,  1.16 step/s, accuracy=0.56, loss=9.99, step=76000]    \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 102.19 uttr/s, accuracy=0.59, loss=10.41]\n",
+      "Train: 100% 2000/2000 [05:18<00:00,  6.29 step/s, accuracy=0.75, loss=9.66, step=78000]  \n",
+      "Valid: 100% 5664/5667 [00:56<00:00, 101.07 uttr/s, accuracy=0.60, loss=10.30]\n",
+      "Train: 100% 2000/2000 [06:04<00:00,  5.49 step/s, accuracy=0.56, loss=11.76, step=8e+4]  \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 101.88 uttr/s, accuracy=0.60, loss=10.43]\n",
       "Train:   0% 0/2000 [00:00<?, ? step/s]"
      ]
     },
@@ -678,23 +731,23 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Step 80000, best model saved. (accuracy=0.8452)\n"
+      "Step 80000, best model saved. (accuracy=0.6042)\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Train: 100% 2000/2000 [04:54<00:00,  6.80 step/s, accuracy=0.94, loss=0.18, step=82000]  \n",
-      "Valid: 100% 5664/5667 [00:54<00:00, 103.49 uttr/s, accuracy=0.84, loss=0.84]\n",
-      "Train: 100% 2000/2000 [05:15<00:00,  6.35 step/s, accuracy=0.91, loss=0.36, step=84000] \n",
-      "Valid: 100% 5664/5667 [00:55<00:00, 101.81 uttr/s, accuracy=0.85, loss=0.82]\n",
-      "Train: 100% 2000/2000 [05:12<00:00,  6.41 step/s, accuracy=0.94, loss=0.25, step=86000] \n",
-      "Valid: 100% 5664/5667 [00:56<00:00, 99.54 uttr/s, accuracy=0.85, loss=0.84] \n",
-      "Train: 100% 2000/2000 [05:35<00:00,  5.96 step/s, accuracy=0.97, loss=0.12, step=88000]  \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 107.23 uttr/s, accuracy=0.86, loss=0.80]\n",
-      "Train: 100% 2000/2000 [05:00<00:00,  6.66 step/s, accuracy=0.97, loss=0.19, step=9e+4]   \n",
-      "Valid: 100% 5664/5667 [00:54<00:00, 104.47 uttr/s, accuracy=0.85, loss=0.85]\n",
+      "Train: 100% 2000/2000 [05:18<00:00,  6.29 step/s, accuracy=0.75, loss=8.42, step=82000]  \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 101.29 uttr/s, accuracy=0.61, loss=10.19]\n",
+      "Train: 100% 2000/2000 [05:18<00:00,  6.28 step/s, accuracy=0.59, loss=10.21, step=84000]\n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 101.42 uttr/s, accuracy=0.60, loss=10.42]\n",
+      "Train: 100% 2000/2000 [05:18<00:00,  6.29 step/s, accuracy=0.78, loss=7.36, step=86000]  \n",
+      "Valid: 100% 5664/5667 [00:56<00:00, 100.79 uttr/s, accuracy=0.61, loss=10.23]\n",
+      "Train: 100% 2000/2000 [06:04<00:00,  5.49 step/s, accuracy=0.75, loss=7.39, step=88000]   \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 101.52 uttr/s, accuracy=0.61, loss=10.19]\n",
+      "Train: 100% 2000/2000 [05:18<00:00,  6.28 step/s, accuracy=0.78, loss=7.21, step=9e+4]   \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 101.38 uttr/s, accuracy=0.61, loss=10.16]\n",
       "Train:   0% 0/2000 [00:00<?, ? step/s]"
      ]
     },
@@ -702,23 +755,23 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Step 90000, best model saved. (accuracy=0.8598)\n"
+      "Step 90000, best model saved. (accuracy=0.6148)\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Train: 100% 2000/2000 [04:59<00:00,  6.68 step/s, accuracy=0.94, loss=0.20, step=92000] \n",
-      "Valid: 100% 5664/5667 [00:55<00:00, 101.47 uttr/s, accuracy=0.85, loss=0.83]\n",
-      "Train: 100% 2000/2000 [06:05<00:00,  5.47 step/s, accuracy=0.91, loss=0.20, step=94000] \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 108.13 uttr/s, accuracy=0.86, loss=0.73]\n",
-      "Train: 100% 2000/2000 [05:17<00:00,  6.29 step/s, accuracy=1.00, loss=0.12, step=96000]  \n",
-      "Valid: 100% 5664/5667 [00:53<00:00, 105.82 uttr/s, accuracy=0.86, loss=0.80]\n",
-      "Train: 100% 2000/2000 [05:10<00:00,  6.45 step/s, accuracy=0.94, loss=0.23, step=98000]  \n",
-      "Valid: 100% 5664/5667 [00:53<00:00, 105.24 uttr/s, accuracy=0.86, loss=0.82]\n",
-      "Train: 100% 2000/2000 [05:13<00:00,  6.39 step/s, accuracy=0.94, loss=0.23, step=1e+5]  \n",
-      "Valid: 100% 5664/5667 [00:56<00:00, 101.03 uttr/s, accuracy=0.86, loss=0.77]\n",
+      "Train: 100% 2000/2000 [05:18<00:00,  6.28 step/s, accuracy=0.72, loss=8.60, step=92000] \n",
+      "Valid: 100% 5664/5667 [00:56<00:00, 101.11 uttr/s, accuracy=0.61, loss=10.13]\n",
+      "Train: 100% 2000/2000 [06:04<00:00,  5.49 step/s, accuracy=0.72, loss=8.10, step=94000]  \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 102.58 uttr/s, accuracy=0.62, loss=10.07]\n",
+      "Train: 100% 2000/2000 [05:18<00:00,  6.29 step/s, accuracy=0.72, loss=8.45, step=96000]   \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 101.82 uttr/s, accuracy=0.62, loss=10.08]\n",
+      "Train: 100% 2000/2000 [05:18<00:00,  6.29 step/s, accuracy=0.69, loss=7.75, step=98000]  \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 101.37 uttr/s, accuracy=0.62, loss=10.08]\n",
+      "Train: 100% 2000/2000 [05:18<00:00,  6.27 step/s, accuracy=0.72, loss=8.97, step=1e+5]  \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 101.33 uttr/s, accuracy=0.63, loss=10.04]\n",
       "Train:   0% 0/2000 [00:00<?, ? step/s]"
      ]
     },
@@ -726,23 +779,23 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Step 100000, best model saved. (accuracy=0.8630)\n"
+      "Step 100000, best model saved. (accuracy=0.6262)\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Train: 100% 2000/2000 [05:58<00:00,  5.57 step/s, accuracy=0.88, loss=0.33, step=102000]\n",
-      "Valid: 100% 5664/5667 [00:53<00:00, 106.35 uttr/s, accuracy=0.86, loss=0.75]\n",
-      "Train: 100% 2000/2000 [05:16<00:00,  6.33 step/s, accuracy=0.94, loss=0.21, step=104000]  \n",
-      "Valid: 100% 5664/5667 [00:53<00:00, 105.24 uttr/s, accuracy=0.86, loss=0.75]\n",
-      "Train: 100% 2000/2000 [05:14<00:00,  6.36 step/s, accuracy=0.97, loss=0.17, step=106000]  \n",
-      "Valid: 100% 5664/5667 [00:55<00:00, 102.57 uttr/s, accuracy=0.87, loss=0.75]\n",
-      "Train: 100% 2000/2000 [05:15<00:00,  6.34 step/s, accuracy=0.97, loss=0.07, step=108000] \n",
-      "Valid: 100% 5664/5667 [00:57<00:00, 99.33 uttr/s, accuracy=0.87, loss=0.73] \n",
-      "Train: 100% 2000/2000 [06:00<00:00,  5.55 step/s, accuracy=0.97, loss=0.11, step=110000] \n",
-      "Valid: 100% 5664/5667 [00:53<00:00, 105.74 uttr/s, accuracy=0.87, loss=0.73]\n",
+      "Train: 100% 2000/2000 [06:04<00:00,  5.49 step/s, accuracy=0.84, loss=9.42, step=102000] \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 102.73 uttr/s, accuracy=0.62, loss=10.01]\n",
+      "Train: 100% 2000/2000 [05:18<00:00,  6.29 step/s, accuracy=0.75, loss=8.01, step=104000]  \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 101.91 uttr/s, accuracy=0.62, loss=9.97]\n",
+      "Train: 100% 2000/2000 [05:18<00:00,  6.29 step/s, accuracy=0.78, loss=8.68, step=106000]  \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 101.60 uttr/s, accuracy=0.63, loss=9.87]\n",
+      "Train: 100% 2000/2000 [05:18<00:00,  6.29 step/s, accuracy=0.72, loss=8.37, step=108000] \n",
+      "Valid: 100% 5664/5667 [00:56<00:00, 101.12 uttr/s, accuracy=0.64, loss=9.75]\n",
+      "Train: 100% 2000/2000 [06:04<00:00,  5.49 step/s, accuracy=0.75, loss=7.60, step=110000] \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 102.22 uttr/s, accuracy=0.63, loss=9.89]\n",
       "Train:   0% 0/2000 [00:00<?, ? step/s]"
      ]
     },
@@ -750,23 +803,23 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Step 110000, best model saved. (accuracy=0.8669)\n"
+      "Step 110000, best model saved. (accuracy=0.6386)\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Train: 100% 2000/2000 [05:11<00:00,  6.42 step/s, accuracy=1.00, loss=0.03, step=112000]  \n",
-      "Valid: 100% 5664/5667 [00:54<00:00, 104.34 uttr/s, accuracy=0.87, loss=0.72]\n",
-      "Train: 100% 2000/2000 [05:13<00:00,  6.37 step/s, accuracy=0.94, loss=0.13, step=114000]  \n",
-      "Valid: 100% 5664/5667 [00:55<00:00, 102.10 uttr/s, accuracy=0.87, loss=0.73]\n",
-      "Train: 100% 2000/2000 [05:28<00:00,  6.10 step/s, accuracy=0.91, loss=0.43, step=116000] \n",
-      "Valid: 100% 5664/5667 [00:56<00:00, 100.56 uttr/s, accuracy=0.87, loss=0.72]\n",
-      "Train: 100% 2000/2000 [06:08<00:00,  5.43 step/s, accuracy=1.00, loss=0.04, step=118000] \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 107.06 uttr/s, accuracy=0.88, loss=0.70]\n",
-      "Train: 100% 2000/2000 [05:18<00:00,  6.28 step/s, accuracy=1.00, loss=0.03, step=120000]  \n",
-      "Valid: 100% 5664/5667 [00:53<00:00, 105.12 uttr/s, accuracy=0.88, loss=0.73]\n",
+      "Train: 100% 2000/2000 [05:18<00:00,  6.29 step/s, accuracy=0.84, loss=7.24, step=112000]   \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 101.93 uttr/s, accuracy=0.63, loss=9.79]\n",
+      "Train: 100% 2000/2000 [05:17<00:00,  6.29 step/s, accuracy=0.78, loss=6.80, step=114000]   \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 101.34 uttr/s, accuracy=0.64, loss=9.72]\n",
+      "Train: 100% 2000/2000 [05:18<00:00,  6.29 step/s, accuracy=0.56, loss=11.41, step=116000]\n",
+      "Valid: 100% 5664/5667 [00:56<00:00, 101.02 uttr/s, accuracy=0.63, loss=9.76]\n",
+      "Train: 100% 2000/2000 [06:04<00:00,  5.49 step/s, accuracy=0.84, loss=6.50, step=118000] \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 102.38 uttr/s, accuracy=0.63, loss=9.70]\n",
+      "Train: 100% 2000/2000 [05:18<00:00,  6.28 step/s, accuracy=0.91, loss=6.47, step=120000]  \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 101.49 uttr/s, accuracy=0.64, loss=9.72]\n",
       "Train:   0% 0/2000 [00:00<?, ? step/s]"
      ]
     },
@@ -774,23 +827,23 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Step 120000, best model saved. (accuracy=0.8773)\n"
+      "Step 120000, best model saved. (accuracy=0.6427)\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Train: 100% 2000/2000 [05:18<00:00,  6.28 step/s, accuracy=1.00, loss=0.06, step=122000]  \n",
-      "Valid: 100% 5664/5667 [00:54<00:00, 103.07 uttr/s, accuracy=0.88, loss=0.68]\n",
-      "Train: 100% 2000/2000 [05:10<00:00,  6.43 step/s, accuracy=0.97, loss=0.07, step=124000] \n",
-      "Valid: 100% 5664/5667 [00:57<00:00, 98.42 uttr/s, accuracy=0.88, loss=0.68] \n",
-      "Train: 100% 2000/2000 [06:17<00:00,  5.29 step/s, accuracy=1.00, loss=0.01, step=126000] \n",
-      "Valid: 100% 5664/5667 [00:53<00:00, 105.35 uttr/s, accuracy=0.88, loss=0.67]\n",
-      "Train: 100% 2000/2000 [05:00<00:00,  6.65 step/s, accuracy=1.00, loss=0.05, step=128000]  \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 106.92 uttr/s, accuracy=0.88, loss=0.66]\n",
-      "Train: 100% 2000/2000 [04:59<00:00,  6.67 step/s, accuracy=1.00, loss=0.05, step=130000]  \n",
-      "Valid: 100% 5664/5667 [00:54<00:00, 103.12 uttr/s, accuracy=0.89, loss=0.65]\n",
+      "Train: 100% 2000/2000 [05:19<00:00,  6.26 step/s, accuracy=0.78, loss=7.12, step=122000]  \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 101.43 uttr/s, accuracy=0.64, loss=9.75]\n",
+      "Train: 100% 2000/2000 [05:18<00:00,  6.29 step/s, accuracy=0.78, loss=8.39, step=124000] \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 101.32 uttr/s, accuracy=0.64, loss=9.69]\n",
+      "Train: 100% 2000/2000 [06:04<00:00,  5.49 step/s, accuracy=0.81, loss=7.61, step=126000] \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 102.44 uttr/s, accuracy=0.64, loss=9.59]\n",
+      "Train: 100% 2000/2000 [05:18<00:00,  6.29 step/s, accuracy=0.69, loss=8.72, step=128000]  \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 101.67 uttr/s, accuracy=0.64, loss=9.58]\n",
+      "Train: 100% 2000/2000 [05:18<00:00,  6.28 step/s, accuracy=0.81, loss=6.57, step=130000]  \n",
+      "Valid: 100% 5664/5667 [00:56<00:00, 100.93 uttr/s, accuracy=0.65, loss=9.67]\n",
       "Train:   0% 0/2000 [00:00<?, ? step/s]"
      ]
     },
@@ -798,23 +851,23 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Step 130000, best model saved. (accuracy=0.8916)\n"
+      "Step 130000, best model saved. (accuracy=0.6455)\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Train: 100% 2000/2000 [05:22<00:00,  6.19 step/s, accuracy=1.00, loss=0.06, step=132000] \n",
-      "Valid: 100% 5664/5667 [00:55<00:00, 101.36 uttr/s, accuracy=0.88, loss=0.66]\n",
-      "Train: 100% 2000/2000 [06:04<00:00,  5.48 step/s, accuracy=0.97, loss=0.15, step=134000] \n",
-      "Valid: 100% 5664/5667 [00:53<00:00, 105.72 uttr/s, accuracy=0.88, loss=0.66]\n",
-      "Train: 100% 2000/2000 [05:19<00:00,  6.25 step/s, accuracy=0.97, loss=0.06, step=136000]  \n",
-      "Valid: 100% 5664/5667 [00:54<00:00, 104.73 uttr/s, accuracy=0.88, loss=0.65]\n",
-      "Train: 100% 2000/2000 [05:16<00:00,  6.32 step/s, accuracy=1.00, loss=0.01, step=138000]  \n",
-      "Valid: 100% 5664/5667 [00:55<00:00, 102.74 uttr/s, accuracy=0.88, loss=0.70]\n",
-      "Train: 100% 2000/2000 [28:15<00:00,  1.18 step/s, accuracy=0.94, loss=0.29, step=140000]   \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 107.28 uttr/s, accuracy=0.89, loss=0.65]\n",
+      "Train: 100% 2000/2000 [05:18<00:00,  6.29 step/s, accuracy=0.69, loss=8.58, step=132000]  \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 101.36 uttr/s, accuracy=0.65, loss=9.56]\n",
+      "Train: 100% 2000/2000 [06:03<00:00,  5.50 step/s, accuracy=0.59, loss=10.18, step=134000] \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 102.10 uttr/s, accuracy=0.64, loss=9.61]\n",
+      "Train: 100% 2000/2000 [05:18<00:00,  6.28 step/s, accuracy=0.69, loss=9.72, step=136000]  \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 101.69 uttr/s, accuracy=0.65, loss=9.55]\n",
+      "Train: 100% 2000/2000 [05:18<00:00,  6.28 step/s, accuracy=0.75, loss=7.29, step=138000]  \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 101.29 uttr/s, accuracy=0.65, loss=9.59]\n",
+      "Train: 100% 2000/2000 [05:18<00:00,  6.27 step/s, accuracy=0.88, loss=6.65, step=140000] \n",
+      "Valid: 100% 5664/5667 [00:56<00:00, 100.97 uttr/s, accuracy=0.65, loss=9.55]\n",
       "Train:   0% 0/2000 [00:00<?, ? step/s]"
      ]
     },
@@ -822,23 +875,23 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Step 140000, best model saved. (accuracy=0.8916)\n"
+      "Step 140000, best model saved. (accuracy=0.6518)\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Train: 100% 2000/2000 [05:14<00:00,  6.35 step/s, accuracy=1.00, loss=0.04, step=142000] \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 108.42 uttr/s, accuracy=0.89, loss=0.64]\n",
-      "Train: 100% 2000/2000 [04:28<00:00,  7.44 step/s, accuracy=1.00, loss=0.02, step=144000]  \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 107.12 uttr/s, accuracy=0.89, loss=0.66]\n",
-      "Train: 100% 2000/2000 [04:29<00:00,  7.43 step/s, accuracy=0.97, loss=0.20, step=146000] \n",
-      "Valid: 100% 5664/5667 [00:54<00:00, 103.30 uttr/s, accuracy=0.89, loss=0.67]\n",
-      "Train: 100% 2000/2000 [29:24<00:00,  1.13 step/s, accuracy=0.97, loss=0.09, step=148000]    \n",
-      "Valid: 100% 5664/5667 [03:18<00:00, 28.55 uttr/s, accuracy=0.89, loss=0.65] \n",
-      "Train: 100% 2000/2000 [05:16<00:00,  6.32 step/s, accuracy=1.00, loss=0.06, step=150000] \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 108.23 uttr/s, accuracy=0.89, loss=0.65]\n",
+      "Train: 100% 2000/2000 [06:04<00:00,  5.49 step/s, accuracy=0.88, loss=7.21, step=142000] \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 102.17 uttr/s, accuracy=0.65, loss=9.53]\n",
+      "Train: 100% 2000/2000 [05:18<00:00,  6.29 step/s, accuracy=0.84, loss=7.21, step=144000]   \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 101.69 uttr/s, accuracy=0.66, loss=9.43]\n",
+      "Train: 100% 2000/2000 [05:18<00:00,  6.28 step/s, accuracy=0.75, loss=7.19, step=146000] \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 101.30 uttr/s, accuracy=0.65, loss=9.56]\n",
+      "Train: 100% 2000/2000 [05:18<00:00,  6.28 step/s, accuracy=0.75, loss=7.84, step=148000] \n",
+      "Valid: 100% 5664/5667 [00:56<00:00, 100.88 uttr/s, accuracy=0.65, loss=9.53]\n",
+      "Train: 100% 2000/2000 [06:05<00:00,  5.48 step/s, accuracy=0.81, loss=6.95, step=150000]  \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 101.93 uttr/s, accuracy=0.65, loss=9.57]\n",
       "Train:   0% 0/2000 [00:00<?, ? step/s]"
      ]
     },
@@ -846,31 +899,38 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Step 150000, best model saved. (accuracy=0.8916)\n"
+      "Step 150000, best model saved. (accuracy=0.6577)\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Train: 100% 2000/2000 [04:29<00:00,  7.41 step/s, accuracy=1.00, loss=0.01, step=152000]  \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 107.46 uttr/s, accuracy=0.88, loss=0.66]\n",
-      "Train: 100% 2000/2000 [04:30<00:00,  7.41 step/s, accuracy=1.00, loss=0.05, step=154000] \n",
-      "Valid: 100% 5664/5667 [00:53<00:00, 106.62 uttr/s, accuracy=0.89, loss=0.65]\n",
-      "Train:  14% 286/2000 [00:32<03:11,  8.97 step/s, accuracy=1.00, loss=0.02, step=154286]"
+      "Train: 100% 2000/2000 [05:18<00:00,  6.29 step/s, accuracy=0.91, loss=5.56, step=152000]  \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 101.50 uttr/s, accuracy=0.65, loss=9.49]\n",
+      "Train: 100% 2000/2000 [05:18<00:00,  6.29 step/s, accuracy=0.66, loss=9.74, step=154000] \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 101.21 uttr/s, accuracy=0.65, loss=9.42]\n",
+      "Train: 100% 2000/2000 [05:18<00:00,  6.28 step/s, accuracy=0.81, loss=8.38, step=156000] \n",
+      "Valid: 100% 5664/5667 [00:56<00:00, 101.05 uttr/s, accuracy=0.66, loss=9.45]\n",
+      "Train: 100% 2000/2000 [06:04<00:00,  5.49 step/s, accuracy=0.81, loss=6.96, step=158000]  \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 102.01 uttr/s, accuracy=0.64, loss=9.62]\n",
+      "Train: 100% 2000/2000 [05:18<00:00,  6.28 step/s, accuracy=0.84, loss=6.30, step=160000]  \n",
+      "Valid: 100% 5664/5667 [00:55<00:00, 101.54 uttr/s, accuracy=0.65, loss=9.55]\n",
+      "Train:   0% 0/2000 [00:00<?, ? step/s]"
      ]
     },
     {
-     "ename": "KeyboardInterrupt",
-     "evalue": "",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
-      "Cell \u001b[0;32mIn[10], line 110\u001b[0m\n\u001b[1;32m    105\u001b[0m     train_log({\n\u001b[1;32m    106\u001b[0m         \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mstatus\u001b[39m\u001b[38;5;124m'\u001b[39m: \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcompleted\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m    107\u001b[0m     })\n\u001b[1;32m    109\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;18m__name__\u001b[39m \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m__main__\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m--> 110\u001b[0m     \u001b[43mmain\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mparse_args\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n",
-      "Cell \u001b[0;32mIn[10], line 63\u001b[0m, in \u001b[0;36mmain\u001b[0;34m(data_dir, save_path, batch_size, n_workers, valid_steps, warmup_steps, total_steps, save_steps)\u001b[0m\n\u001b[1;32m     60\u001b[0m     train_iterator \u001b[38;5;241m=\u001b[39m \u001b[38;5;28miter\u001b[39m(train_loader)\n\u001b[1;32m     61\u001b[0m     batch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mnext\u001b[39m(train_iterator)\n\u001b[0;32m---> 63\u001b[0m loss, accuracy \u001b[38;5;241m=\u001b[39m \u001b[43mmodel_fn\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbatch\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcriterion\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdevice\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     64\u001b[0m batch_loss \u001b[38;5;241m=\u001b[39m loss\u001b[38;5;241m.\u001b[39mitem()\n\u001b[1;32m     65\u001b[0m batch_accuracy \u001b[38;5;241m=\u001b[39m accuracy\u001b[38;5;241m.\u001b[39mitem()\n",
-      "Cell \u001b[0;32mIn[8], line 8\u001b[0m, in \u001b[0;36mmodel_fn\u001b[0;34m(batch, model, criterion, device)\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Forward a batch through the model.\"\"\"\u001b[39;00m\n\u001b[1;32m      7\u001b[0m mels, labels \u001b[38;5;241m=\u001b[39m batch\n\u001b[0;32m----> 8\u001b[0m mels \u001b[38;5;241m=\u001b[39m \u001b[43mmels\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mto\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdevice\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m      9\u001b[0m labels \u001b[38;5;241m=\u001b[39m labels\u001b[38;5;241m.\u001b[39mto(device)\n\u001b[1;32m     11\u001b[0m outs, outs_length \u001b[38;5;241m=\u001b[39m model(mels)\n",
-      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Step 160000, best model saved. (accuracy=0.6649)\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Train:   0% 0/2000 [00:00<?, ? step/s]\n"
      ]
     }
    ],
@@ -881,7 +941,7 @@
     "import torch.nn as nn\n",
     "from torch.optim import AdamW\n",
     "from torch.utils.data import DataLoader, random_split\n",
-    "\n",
+    "import gc\n",
     "def parse_args():\n",
     "    \"\"\"arguments\"\"\"\n",
     "    config = {\n",
@@ -919,7 +979,9 @@
     "\n",
     "    model = Classifier(n_spks=speaker_num).to(device)\n",
     "    #model.load_state_dict(torch.load('./model.ckpt'))\n",
-    "    criterion = nn.CrossEntropyLoss()\n",
+    "    #criterion = nn.CrossEntropyLoss()\n",
+    "    criterion = AdMSoftmaxLoss(40, speaker_num, s=30.0, m=0.4).to(device) # Default values recommended by\n",
+    "    # “Additive Margin Softmax for Face Verification.” Wang, Feng, Jian Cheng, Weiyang Liu and Haijun Liu. IEEE Signal Processing Letters 25 (2018): 926-930.\n",
     "    optimizer = AdamW(model.parameters(), lr=1e-3)\n",
     "    scheduler = get_cosine_schedule_with_warmup(optimizer, warmup_steps, total_steps)\n",
     "    print(f\"[Info]: Finish creating model!\",flush = True)\n",
@@ -982,23 +1044,14 @@
     "    train_log({\n",
     "        'status': 'completed'\n",
     "    })\n",
+    "\n",
+    "    del train_loader, valid_loader\n",
+    "    gc.collect()\n",
     "    \n",
     "if __name__ == \"__main__\":\n",
     "    main(**parse_args())"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import gc\n",
-    "\n",
-    "#del train_loader, valid_loader\n",
-    "gc.collect()"
-   ]
-  },
   {
    "cell_type": "markdown",
    "metadata": {
@@ -1012,7 +1065,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 12,
    "metadata": {
     "id": "efS4pCmAJXJH"
    },
@@ -1039,7 +1092,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
    "metadata": {
     "colab": {
      "base_uri": "https://localhost:8080/",
@@ -1074,7 +1127,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "94406e7fe0a64bb187b1edb64c2c6c4a",
+       "model_id": "a6e9118fb3ff4e96a988a7df19ca7b98",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1155,21 +1208,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "ename": "NameError",
-     "evalue": "name 'dataloader' is not defined",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
-      "Cell \u001b[0;32mIn[7], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mgc\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m \u001b[38;5;28;01mdel\u001b[39;00m dataloader\n\u001b[1;32m      3\u001b[0m gc\u001b[38;5;241m.\u001b[39mcollect()\n",
-      "\u001b[0;31mNameError\u001b[0m: name 'dataloader' is not defined"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "import gc\n",
     "del dataloader\n",
@@ -1181,7 +1222,9 @@
    "execution_count": null,
    "metadata": {},
    "outputs": [],
-   "source": []
+   "source": [
+    "\n"
+   ]
   }
  ],
  "metadata": {