20230328 public 0.945 convformer so powerful

ML2023_hw04.ipynb

@@ -61,9 +61,65 @@
     "id": "U6Y1cfpDfpON",
     "outputId": "6ba26637-5c7b-48a9-be0b-1f10ba76590a"
    },
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "TERM_PROGRAM=Apple_Terminal\r\n",
+      "SHELL=/bin/zsh\r\n",
+      "TERM=xterm-color\r\n",
+      "TMPDIR=/var/folders/_h/h6dll3ns6lvc3ljhm8wlnp7h0000gn/T/\r\n",
+      "TERM_PROGRAM_VERSION=447\r\n",
+      "TERM_SESSION_ID=643AED00-1886-4C0F-8047-3E99B4468B4D\r\n",
+      "USER=sam\r\n",
+      "SSH_AUTH_SOCK=/private/tmp/com.apple.launchd.H0LZFasLIa/Listeners\r\n",
+      "PATH=/Users/sam/my_venv/ml/bin:/opt/homebrew/opt/bison/bin:/opt/homebrew/opt/cbc/bin:/Users/sam/flutter/bin:/Library/Frameworks/Python.framework/Versions/3.10/bin:/opt/homebrew/bin:/opt/homebrew/sbin:/usr/local/bin:/System/Cryptexes/App/usr/bin:/usr/bin:/bin:/usr/sbin:/sbin:/opt/X11/bin:/Library/Apple/usr/bin:/Applications/Wireshark.app/Contents/MacOS:/var/run/com.apple.security.cryptexd/codex.system/bootstrap/usr/local/bin:/var/run/com.apple.security.cryptexd/codex.system/bootstrap/usr/bin:/var/run/com.apple.security.cryptexd/codex.system/bootstrap/usr/appleinternal/bin\r\n",
+      "LaunchInstanceID=5020678A-B21F-4CCA-B898-3F054FE219CA\r\n",
+      "__CFBundleIdentifier=com.apple.Terminal\r\n",
+      "PWD=/Users/sam/Documents/ML_2023_Spring/HW4\r\n",
+      "XPC_FLAGS=0x0\r\n",
+      "XPC_SERVICE_NAME=0\r\n",
+      "SHLVL=1\r\n",
+      "HOME=/Users/sam\r\n",
+      "LOGNAME=sam\r\n",
+      "DISPLAY=/private/tmp/com.apple.launchd.jaQhA1Fj8r/org.xquartz:0\r\n",
+      "SECURITYSESSIONID=186ac\r\n",
+      "OLDPWD=/Users/sam/Documents/ML_2023_Spring/HW4\r\n",
+      "HOMEBREW_PREFIX=/opt/homebrew\r\n",
+      "HOMEBREW_CELLAR=/opt/homebrew/Cellar\r\n",
+      "HOMEBREW_REPOSITORY=/opt/homebrew\r\n",
+      "MANPATH=/opt/homebrew/share/man::\r\n",
+      "INFOPATH=/opt/homebrew/share/info:\r\n",
+      "ZSH=/Users/sam/.oh-my-zsh\r\n",
+      "PAGER=cat\r\n",
+      "LESS=-R\r\n",
+      "LSCOLORS=Gxfxcxdxbxegedabagacad\r\n",
+      "VIRTUAL_ENV=/Users/sam/my_venv/ml\r\n",
+      "PS1=(ml) %(?:%{\u001b[01;32m%}➜ :%{\u001b[01;31m%}➜ ) %{$fg[cyan]%}%c%{$reset_color%} $(git_prompt_info)\r\n",
+      "VIRTUAL_ENV_PROMPT=(ml) \r\n",
+      "MONGODB_HOSTNAME=192.168.255.5\r\n",
+      "MONGODB_PORT=27717\r\n",
+      "MONGODB_USERNAME=sam\r\n",
+      "MONGODB_PASSWORD=pc152pc152\r\n",
+      "PYTORCH_ENABLE_MPS_FALLBACK=1\r\n",
+      "LANG=zh_TW.UTF-8\r\n",
+      "__CF_USER_TEXT_ENCODING=0x1F5:0x2:0x35\r\n",
+      "PYDEVD_USE_FRAME_EVAL=NO\r\n",
+      "JPY_PARENT_PID=5926\r\n",
+      "CLICOLOR=1\r\n",
+      "FORCE_COLOR=1\r\n",
+      "CLICOLOR_FORCE=1\r\n",
+      "GIT_PAGER=cat\r\n",
+      "MPLBACKEND=module://matplotlib_inline.backend_inline\r\n",
+      "_=/usr/bin/printenv\r\n"
+     ]
+    }
+   ],
    "source": [
-    "#!tar zxf Dataset.tar.gz"
+    "#!tar zxf Dataset.tar.gz\n",
+    "!export PYTORCH_ENABLE_MPS_FALLBACK=1\n",
+    "!printenv"
    ]
   },
   {
@@ -77,7 +133,7 @@
     "import numpy as np\n",
     "import torch\n",
     "import random\n",
-    "from common_function import train_start_log, train_complete_log\n",
+    "from common_function import train_start_log, train_log\n",
     "\n",
     "def set_seed(seed):\n",
     "    np.random.seed(seed)\n",
@@ -86,6 +142,8 @@
     "    if torch.cuda.is_available():\n",
     "        torch.cuda.manual_seed(seed)\n",
     "        torch.cuda.manual_seed_all(seed)\n",
+    "        torch.mps.manual_seed(seed)\n",
+    "        torch.mps.seed(seed)\n",
     "    torch.backends.cudnn.benchmark = False\n",
     "    torch.backends.cudnn.deterministic = True\n",
     "\n",
@@ -243,7 +301,7 @@
     "import torch\n",
     "import torch.nn as nn\n",
     "import torch.nn.functional as F\n",
-    "\n",
+    "import torchaudio\n",
     "\n",
     "class Classifier(nn.Module):\n",
     "    def __init__(self, d_model=160, n_spks=600, dropout=0.1):\n",
@@ -253,17 +311,14 @@
     "        # TODO:\n",
     "        #   Change Transformer to Conformer.\n",
     "        #   https://arxiv.org/abs/2005.08100\n",
-    "        self.encoder_layer = nn.TransformerEncoderLayer(\n",
-    "            d_model=d_model, dim_feedforward=256, nhead=8\n",
-    "        )\n",
-    "        self.encoder = nn.TransformerEncoder(self.encoder_layer, num_layers=2)\n",
-    "\n",
+    "        #self.encoder_layer = nn.TransformerEncoderLayer(\n",
+    "        #    d_model=d_model, dim_feedforward=256, nhead=8\n",
+    "        #)\n",
+    "        #self.encoder = nn.TransformerEncoder(self.encoder_layer, num_layers=2)\n",
+    "        self.encoder = torchaudio.models.Conformer(d_model, 8, 256, 2, 31) # From torchaudio\n",
+    "        \n",
     "        # Project the the dimension of features from d_model into speaker nums.\n",
-    "        self.pred_layer = nn.Sequential(\n",
-    "            nn.Linear(d_model, d_model),\n",
-    "            nn.Sigmoid(),\n",
-    "            nn.Linear(d_model, n_spks),\n",
-    "        )\n",
+    "        self.pred_layer = nn.Linear(d_model, n_spks)\n",
     "\n",
     "    def forward(self, mels):\n",
     "        \"\"\"\n",
@@ -275,17 +330,22 @@
     "        # out: (batch size, length, d_model)\n",
     "        out = self.prenet(mels)\n",
     "        # out: (length, batch size, d_model)\n",
-    "        out = out.permute(1, 0, 2)\n",
+    "        #out = out.permute(1, 0, 2)\n",
     "        # The encoder layer expect features in the shape of (length, batch size, d_model).\n",
-    "        out = self.encoder(out)\n",
+    "        device = \"cuda\" if torch.cuda.is_available() else 'mps' if torch.backends.mps.is_available() else 'cpu'\n",
+    "        #lengths = torch.randint(out.size(1), (int(out.size(0)),)).to(device)\n",
+    "        #lengths[torch.argmax(lengths)] = out.size(1)\n",
+    "        lengths = torch.full((int(out.size(0)),), out.size(1)).to(device)\n",
+    "        out, _ = self.encoder(out, lengths)\n",
     "        # out: (batch size, length, d_model)\n",
-    "        out = out.transpose(0, 1)\n",
+    "        #out = out.transpose(0, 1)\n",
     "        # mean pooling\n",
-    "        stats = out.mean(dim=1)\n",
+    "        #stats = out.mean(dim=1)\n",
     "\n",
     "        # out: (batch, n_spks)\n",
+    "        stats = out.mean(dim=1)\n",
     "        out = self.pred_layer(stats)\n",
-    "        return out"
+    "        return out, _"
    ]
   },
   {
@@ -388,8 +448,8 @@
     "    mels = mels.to(device)\n",
     "    labels = labels.to(device)\n",
     "\n",
-    "    outs = model(mels)\n",
-    "\n",
+    "    outs, outs_length = model(mels)\n",
+    "    \n",
     "    loss = criterion(outs, labels)\n",
     "\n",
     "    # Get the speaker id with highest probability.\n",
@@ -459,7 +519,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 8,
    "metadata": {
     "colab": {
      "base_uri": "https://localhost:8080/"
@@ -481,480 +541,19 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Train: 100% 2000/2000 [03:09<00:00, 10.54 step/s, accuracy=0.59, loss=1.64, step=2000]\n",
-      "Valid: 100% 5664/5667 [00:50<00:00, 111.30 uttr/s, accuracy=0.61, loss=1.70] \n",
-      "Train: 100% 2000/2000 [03:06<00:00, 10.70 step/s, accuracy=0.62, loss=1.42, step=4000]  \n",
-      "Valid: 100% 5664/5667 [00:51<00:00, 109.66 uttr/s, accuracy=0.61, loss=1.70] \n",
-      "Train: 100% 2000/2000 [03:09<00:00, 10.58 step/s, accuracy=0.75, loss=1.17, step=6000] \n",
-      "Valid: 100% 5664/5667 [00:53<00:00, 106.36 uttr/s, accuracy=0.61, loss=1.73] \n",
-      "Train: 100% 2000/2000 [03:46<00:00,  8.81 step/s, accuracy=0.75, loss=1.15, step=8000] \n",
-      "Valid: 100% 5664/5667 [00:50<00:00, 112.63 uttr/s, accuracy=0.61, loss=1.74] \n",
-      "Train: 100% 2000/2000 [03:01<00:00, 11.02 step/s, accuracy=0.62, loss=1.49, step=1e+4]\n",
-      "Valid: 100% 5664/5667 [00:51<00:00, 109.64 uttr/s, accuracy=0.61, loss=1.71] \n",
-      "Train:   0% 2/2000 [00:00<04:53,  6.81 step/s, accuracy=0.44, loss=2.35, step=1e+4]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Step 10000, best model saved. (accuracy=0.6148)\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Train: 100% 2000/2000 [06:48<00:00,  4.90 step/s, accuracy=0.75, loss=1.41, step=12000]   \n",
-      "Valid: 100% 5664/5667 [16:06<00:00,  5.86 uttr/s, accuracy=0.61, loss=1.70]  \n",
-      "Train: 100% 2000/2000 [02:56<00:00, 11.32 step/s, accuracy=0.62, loss=1.82, step=14000] \n",
-      "Valid: 100% 5664/5667 [00:50<00:00, 112.22 uttr/s, accuracy=0.62, loss=1.67] \n",
-      "Train: 100% 2000/2000 [53:33<00:00,  1.61s/ step, accuracy=0.62, loss=1.24, step=16000]    \n",
-      "Valid: 100% 5664/5667 [00:49<00:00, 113.71 uttr/s, accuracy=0.61, loss=1.69] \n",
-      "Train: 100% 2000/2000 [06:01<00:00,  5.53 step/s, accuracy=0.66, loss=1.28, step=18000]  \n",
-      "Valid: 100% 5664/5667 [00:51<00:00, 110.69 uttr/s, accuracy=0.60, loss=1.71] \n",
-      "Train: 100% 2000/2000 [03:03<00:00, 10.87 step/s, accuracy=0.75, loss=1.24, step=2e+4]   \n",
-      "Valid: 100% 5664/5667 [00:50<00:00, 111.59 uttr/s, accuracy=0.63, loss=1.61] \n",
-      "Train:   0% 2/2000 [00:00<04:40,  7.12 step/s, accuracy=0.72, loss=0.98, step=2e+4]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Step 20000, best model saved. (accuracy=0.6289)\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Train: 100% 2000/2000 [03:03<00:00, 10.92 step/s, accuracy=0.62, loss=1.45, step=22000] \n",
-      "Valid: 100% 5664/5667 [00:51<00:00, 110.00 uttr/s, accuracy=0.62, loss=1.64] \n",
-      "Train: 100% 2000/2000 [03:48<00:00,  8.75 step/s, accuracy=0.47, loss=2.43, step=24000] \n",
-      "Valid: 100% 5664/5667 [00:50<00:00, 112.45 uttr/s, accuracy=0.44, loss=2.66] \n",
-      "Train: 100% 2000/2000 [03:03<00:00, 10.92 step/s, accuracy=0.78, loss=1.01, step=26000]\n",
-      "Valid: 100% 5664/5667 [00:50<00:00, 111.25 uttr/s, accuracy=0.63, loss=1.61] \n",
-      "Train: 100% 2000/2000 [03:07<00:00, 10.66 step/s, accuracy=0.72, loss=1.11, step=28000]  \n",
-      "Valid: 100% 5664/5667 [00:51<00:00, 109.39 uttr/s, accuracy=0.62, loss=1.64] \n",
-      "Train: 100% 2000/2000 [03:07<00:00, 10.68 step/s, accuracy=0.78, loss=0.81, step=3e+4]  \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 108.07 uttr/s, accuracy=0.65, loss=1.57] \n",
-      "Train:   0% 2/2000 [00:00<04:39,  7.16 step/s, accuracy=0.81, loss=0.63, step=3e+4]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Step 30000, best model saved. (accuracy=0.6457)\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Train: 100% 2000/2000 [03:50<00:00,  8.70 step/s, accuracy=0.84, loss=0.77, step=32000] \n",
-      "Valid: 100% 5664/5667 [00:50<00:00, 111.98 uttr/s, accuracy=0.64, loss=1.59] \n",
-      "Train: 100% 2000/2000 [03:03<00:00, 10.87 step/s, accuracy=0.69, loss=1.25, step=34000]  \n",
-      "Valid: 100% 5664/5667 [00:51<00:00, 110.25 uttr/s, accuracy=0.63, loss=1.58] \n",
-      "Train: 100% 2000/2000 [03:00<00:00, 11.05 step/s, accuracy=0.50, loss=1.50, step=36000]  \n",
-      "Valid: 100% 5664/5667 [00:51<00:00, 109.65 uttr/s, accuracy=0.64, loss=1.56] \n",
-      "Train: 100% 2000/2000 [03:03<00:00, 10.89 step/s, accuracy=0.62, loss=1.40, step=38000] \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 107.96 uttr/s, accuracy=0.64, loss=1.60] \n",
-      "Train: 100% 2000/2000 [09:52<00:00,  3.38 step/s, accuracy=0.62, loss=1.35, step=4e+4]   \n",
-      "Valid: 100% 5664/5667 [00:55<00:00, 101.16 uttr/s, accuracy=0.64, loss=1.52]\n",
-      "Train:   0% 0/2000 [00:00<?, ? step/s]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Step 40000, best model saved. (accuracy=0.6457)\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Train: 100% 2000/2000 [04:21<00:00,  7.66 step/s, accuracy=0.81, loss=1.04, step=42000]  \n",
-      "Valid: 100% 5664/5667 [00:55<00:00, 101.18 uttr/s, accuracy=0.63, loss=1.61]\n",
-      "Train: 100% 2000/2000 [04:29<00:00,  7.42 step/s, accuracy=0.81, loss=0.85, step=44000]  \n",
-      "Valid: 100% 5664/5667 [00:56<00:00, 99.96 uttr/s, accuracy=0.65, loss=1.51] \n",
-      "Train: 100% 2000/2000 [04:34<00:00,  7.29 step/s, accuracy=0.50, loss=2.01, step=46000] \n",
-      "Valid: 100% 5664/5667 [00:56<00:00, 100.25 uttr/s, accuracy=0.61, loss=1.70]\n",
-      "Train: 100% 2000/2000 [05:14<00:00,  6.36 step/s, accuracy=0.81, loss=0.64, step=48000] \n",
-      "Valid: 100% 5664/5667 [00:56<00:00, 99.91 uttr/s, accuracy=0.65, loss=1.52] \n",
-      "Train: 100% 2000/2000 [04:24<00:00,  7.55 step/s, accuracy=0.78, loss=0.98, step=5e+4]   \n",
-      "Valid: 100% 5664/5667 [00:55<00:00, 101.81 uttr/s, accuracy=0.65, loss=1.51]\n",
-      "Train:   0% 2/2000 [00:00<05:27,  6.11 step/s, accuracy=0.88, loss=0.51, step=5e+4]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Step 50000, best model saved. (accuracy=0.6543)\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Train: 100% 2000/2000 [04:25<00:00,  7.53 step/s, accuracy=0.78, loss=0.76, step=52000] \n",
-      "Valid: 100% 5664/5667 [00:55<00:00, 101.31 uttr/s, accuracy=0.67, loss=1.45]\n",
-      "Train: 100% 2000/2000 [04:22<00:00,  7.63 step/s, accuracy=0.81, loss=0.99, step=54000] \n",
-      "Valid: 100% 5664/5667 [00:56<00:00, 99.69 uttr/s, accuracy=0.66, loss=1.45] \n",
-      "Train: 100% 2000/2000 [05:04<00:00,  6.56 step/s, accuracy=0.75, loss=0.96, step=56000] \n",
-      "Valid: 100% 5664/5667 [00:55<00:00, 101.28 uttr/s, accuracy=0.67, loss=1.45]\n",
-      "Train: 100% 2000/2000 [04:25<00:00,  7.54 step/s, accuracy=0.69, loss=1.06, step=58000]  \n",
-      "Valid: 100% 5664/5667 [00:55<00:00, 101.55 uttr/s, accuracy=0.66, loss=1.45]\n",
-      "Train: 100% 2000/2000 [04:18<00:00,  7.74 step/s, accuracy=0.72, loss=0.92, step=6e+4]  \n",
-      "Valid: 100% 5664/5667 [01:20<00:00, 70.68 uttr/s, accuracy=0.67, loss=1.43]  \n",
-      "Train:   0% 2/2000 [00:00<05:23,  6.17 step/s, accuracy=0.75, loss=0.91, step=6e+4]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Step 60000, best model saved. (accuracy=0.6713)\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Train: 100% 2000/2000 [1:52:11<00:00,  3.37s/ step, accuracy=0.72, loss=0.88, step=62000]    \n",
-      "Valid: 100% 5664/5667 [05:24<00:00, 17.47 uttr/s, accuracy=0.66, loss=1.48]  \n",
-      "Train: 100% 2000/2000 [03:49<00:00,  8.73 step/s, accuracy=0.78, loss=1.21, step=64000] \n",
-      "Valid: 100% 5664/5667 [00:50<00:00, 112.98 uttr/s, accuracy=0.67, loss=1.46] \n",
-      "Train: 100% 2000/2000 [03:01<00:00, 11.00 step/s, accuracy=0.84, loss=0.86, step=66000]  \n",
-      "Valid: 100% 5664/5667 [00:50<00:00, 112.34 uttr/s, accuracy=0.68, loss=1.40] \n",
-      "Train: 100% 2000/2000 [03:01<00:00, 11.03 step/s, accuracy=0.72, loss=1.02, step=68000] \n",
-      "Valid: 100% 5664/5667 [00:50<00:00, 111.45 uttr/s, accuracy=0.67, loss=1.43] \n",
-      "Train: 100% 2000/2000 [03:00<00:00, 11.06 step/s, accuracy=0.72, loss=1.39, step=7e+4]  \n",
-      "Valid: 100% 5664/5667 [00:51<00:00, 110.38 uttr/s, accuracy=0.69, loss=1.38] \n",
-      "Train:   0% 2/2000 [00:00<04:41,  7.11 step/s, accuracy=0.84, loss=0.73, step=7e+4]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Step 70000, best model saved. (accuracy=0.6896)\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Train: 100% 2000/2000 [03:47<00:00,  8.80 step/s, accuracy=0.69, loss=1.05, step=72000] \n",
-      "Valid: 100% 5664/5667 [00:50<00:00, 112.48 uttr/s, accuracy=0.68, loss=1.38] \n",
-      "Train: 100% 2000/2000 [03:01<00:00, 11.05 step/s, accuracy=0.72, loss=1.14, step=74000]  \n",
-      "Valid: 100% 5664/5667 [00:51<00:00, 110.86 uttr/s, accuracy=0.69, loss=1.37] \n",
-      "Train: 100% 2000/2000 [03:01<00:00, 11.04 step/s, accuracy=0.75, loss=0.93, step=76000] \n",
-      "Valid: 100% 5664/5667 [00:51<00:00, 109.91 uttr/s, accuracy=0.69, loss=1.33] \n",
-      "Train: 100% 2000/2000 [03:02<00:00, 10.97 step/s, accuracy=0.81, loss=0.64, step=78000] \n",
-      "Valid: 100% 5664/5667 [00:51<00:00, 109.28 uttr/s, accuracy=0.68, loss=1.39] \n",
-      "Train: 100% 2000/2000 [03:49<00:00,  8.72 step/s, accuracy=0.75, loss=1.18, step=8e+4] \n",
-      "Valid: 100% 5664/5667 [00:50<00:00, 111.48 uttr/s, accuracy=0.68, loss=1.37] \n",
-      "Train:   0% 2/2000 [00:00<04:14,  7.84 step/s, accuracy=0.78, loss=0.73, step=8e+4]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Step 80000, best model saved. (accuracy=0.6916)\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Train: 100% 2000/2000 [03:03<00:00, 10.91 step/s, accuracy=0.75, loss=0.94, step=82000]  \n",
-      "Valid: 100% 5664/5667 [00:51<00:00, 110.45 uttr/s, accuracy=0.69, loss=1.36] \n",
-      "Train: 100% 2000/2000 [03:03<00:00, 10.92 step/s, accuracy=0.84, loss=0.58, step=84000] \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 108.73 uttr/s, accuracy=0.69, loss=1.31] \n",
-      "Train: 100% 2000/2000 [03:03<00:00, 10.90 step/s, accuracy=0.88, loss=0.64, step=86000] \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 108.61 uttr/s, accuracy=0.71, loss=1.30] \n",
-      "Train: 100% 2000/2000 [03:48<00:00,  8.74 step/s, accuracy=0.88, loss=0.77, step=88000]\n",
-      "Valid: 100% 5664/5667 [00:50<00:00, 111.64 uttr/s, accuracy=0.70, loss=1.32] \n",
-      "Train: 100% 2000/2000 [03:01<00:00, 11.03 step/s, accuracy=0.84, loss=0.75, step=9e+4]   \n",
-      "Valid: 100% 5664/5667 [00:51<00:00, 110.71 uttr/s, accuracy=0.71, loss=1.29] \n",
-      "Train:   0% 2/2000 [00:00<05:07,  6.51 step/s, accuracy=0.88, loss=0.69, step=9e+4]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Step 90000, best model saved. (accuracy=0.7145)\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Train: 100% 2000/2000 [03:01<00:00, 11.02 step/s, accuracy=0.94, loss=0.60, step=92000] \n",
-      "Valid: 100% 5664/5667 [00:51<00:00, 109.85 uttr/s, accuracy=0.71, loss=1.26] \n",
-      "Train: 100% 2000/2000 [03:47<00:00,  8.79 step/s, accuracy=0.66, loss=1.01, step=94000] \n",
-      "Valid: 100% 5664/5667 [00:50<00:00, 112.85 uttr/s, accuracy=0.71, loss=1.25] \n",
-      "Train: 100% 2000/2000 [03:01<00:00, 11.02 step/s, accuracy=0.81, loss=0.70, step=96000]\n",
-      "Valid: 100% 5664/5667 [00:50<00:00, 111.18 uttr/s, accuracy=0.71, loss=1.28] \n",
-      "Train: 100% 2000/2000 [03:01<00:00, 11.02 step/s, accuracy=0.88, loss=0.97, step=98000]  \n",
-      "Valid: 100% 5664/5667 [00:51<00:00, 110.06 uttr/s, accuracy=0.72, loss=1.22] \n",
-      "Train: 100% 2000/2000 [03:01<00:00, 11.01 step/s, accuracy=0.84, loss=0.66, step=1e+5]  \n",
-      "Valid: 100% 5664/5667 [00:51<00:00, 109.19 uttr/s, accuracy=0.71, loss=1.26] \n",
-      "Train:   0% 2/2000 [00:00<04:50,  6.88 step/s, accuracy=0.78, loss=0.66, step=1e+5]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Step 100000, best model saved. (accuracy=0.7156)\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Train: 100% 2000/2000 [03:47<00:00,  8.79 step/s, accuracy=0.81, loss=0.59, step=102000]\n",
-      "Valid: 100% 5664/5667 [00:50<00:00, 112.58 uttr/s, accuracy=0.72, loss=1.22] \n",
-      "Train: 100% 2000/2000 [03:01<00:00, 11.00 step/s, accuracy=0.84, loss=0.56, step=104000]  \n",
-      "Valid: 100% 5664/5667 [00:50<00:00, 111.12 uttr/s, accuracy=0.72, loss=1.23] \n",
-      "Train: 100% 2000/2000 [03:01<00:00, 11.03 step/s, accuracy=0.75, loss=1.00, step=106000]  \n",
-      "Valid: 100% 5664/5667 [00:51<00:00, 109.72 uttr/s, accuracy=0.72, loss=1.21] \n",
-      "Train: 100% 2000/2000 [03:01<00:00, 11.01 step/s, accuracy=0.75, loss=0.71, step=108000] \n",
-      "Valid: 100% 5664/5667 [00:51<00:00, 109.17 uttr/s, accuracy=0.72, loss=1.22] \n",
-      "Train: 100% 2000/2000 [03:47<00:00,  8.78 step/s, accuracy=0.91, loss=0.63, step=110000] \n",
-      "Valid: 100% 5664/5667 [00:50<00:00, 112.53 uttr/s, accuracy=0.72, loss=1.22] \n",
-      "Train:   0% 2/2000 [00:00<04:43,  7.06 step/s, accuracy=0.81, loss=0.73, step=110002]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Step 110000, best model saved. (accuracy=0.7209)\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Train: 100% 2000/2000 [03:01<00:00, 10.99 step/s, accuracy=0.84, loss=0.69, step=112000]\n",
-      "Valid: 100% 5664/5667 [00:50<00:00, 111.08 uttr/s, accuracy=0.72, loss=1.20] \n",
-      "Train: 100% 2000/2000 [03:01<00:00, 11.02 step/s, accuracy=0.81, loss=0.83, step=114000]  \n",
-      "Valid: 100% 5664/5667 [00:51<00:00, 109.81 uttr/s, accuracy=0.73, loss=1.18] \n",
-      "Train: 100% 2000/2000 [03:01<00:00, 11.01 step/s, accuracy=0.72, loss=1.14, step=116000] \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 108.72 uttr/s, accuracy=0.72, loss=1.18] \n",
-      "Train: 100% 2000/2000 [03:47<00:00,  8.78 step/s, accuracy=0.84, loss=0.72, step=118000] \n",
-      "Valid: 100% 5664/5667 [00:50<00:00, 112.26 uttr/s, accuracy=0.74, loss=1.14] \n",
-      "Train: 100% 2000/2000 [03:01<00:00, 11.02 step/s, accuracy=0.88, loss=0.58, step=120000]  \n",
-      "Valid: 100% 5664/5667 [00:51<00:00, 110.30 uttr/s, accuracy=0.74, loss=1.15] \n",
-      "Train:   0% 2/2000 [00:00<04:53,  6.80 step/s, accuracy=0.88, loss=0.70, step=120002]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Step 120000, best model saved. (accuracy=0.7387)\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Train: 100% 2000/2000 [03:01<00:00, 11.02 step/s, accuracy=0.81, loss=0.79, step=122000]  \n",
-      "Valid: 100% 5664/5667 [00:51<00:00, 109.55 uttr/s, accuracy=0.73, loss=1.14] \n",
-      "Train: 100% 2000/2000 [03:02<00:00, 10.97 step/s, accuracy=0.91, loss=0.81, step=124000] \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 108.22 uttr/s, accuracy=0.74, loss=1.14] \n",
-      "Train: 100% 2000/2000 [03:47<00:00,  8.78 step/s, accuracy=0.81, loss=0.64, step=126000] \n",
-      "Valid: 100% 5664/5667 [00:50<00:00, 112.00 uttr/s, accuracy=0.73, loss=1.15] \n",
-      "Train: 100% 2000/2000 [03:01<00:00, 11.01 step/s, accuracy=0.88, loss=0.50, step=128000]  \n",
-      "Valid: 100% 5664/5667 [00:51<00:00, 110.37 uttr/s, accuracy=0.74, loss=1.13] \n",
-      "Train: 100% 2000/2000 [03:01<00:00, 11.02 step/s, accuracy=0.91, loss=0.35, step=130000]  \n",
-      "Valid: 100% 5664/5667 [00:51<00:00, 109.04 uttr/s, accuracy=0.74, loss=1.11] \n",
-      "Train:   0% 2/2000 [00:00<05:04,  6.57 step/s, accuracy=0.91, loss=0.70, step=130002]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Step 130000, best model saved. (accuracy=0.7449)\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Train: 100% 2000/2000 [03:01<00:00, 11.00 step/s, accuracy=0.94, loss=0.47, step=132000] \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 108.35 uttr/s, accuracy=0.75, loss=1.08] \n",
-      "Train: 100% 2000/2000 [03:47<00:00,  8.79 step/s, accuracy=0.88, loss=0.54, step=134000] \n",
-      "Valid: 100% 5664/5667 [00:50<00:00, 111.59 uttr/s, accuracy=0.75, loss=1.10] \n",
-      "Train: 100% 2000/2000 [03:06<00:00, 10.74 step/s, accuracy=0.81, loss=0.57, step=136000]  \n",
-      "Valid: 100% 5664/5667 [00:51<00:00, 110.18 uttr/s, accuracy=0.75, loss=1.12] \n",
-      "Train: 100% 2000/2000 [03:02<00:00, 10.96 step/s, accuracy=0.91, loss=0.52, step=138000]  \n",
-      "Valid: 100% 5664/5667 [00:51<00:00, 109.58 uttr/s, accuracy=0.75, loss=1.08] \n",
-      "Train: 100% 2000/2000 [03:02<00:00, 10.94 step/s, accuracy=0.88, loss=0.62, step=140000] \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 108.00 uttr/s, accuracy=0.76, loss=1.08] \n",
-      "Train:   0% 2/2000 [00:00<04:19,  7.69 step/s, accuracy=0.91, loss=0.48, step=140002]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Step 140000, best model saved. (accuracy=0.7569)\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Train: 100% 2000/2000 [03:49<00:00,  8.71 step/s, accuracy=0.81, loss=0.77, step=142000] \n",
-      "Valid: 100% 5664/5667 [00:51<00:00, 110.42 uttr/s, accuracy=0.76, loss=1.08] \n",
-      "Train: 100% 2000/2000 [03:11<00:00, 10.42 step/s, accuracy=0.84, loss=0.71, step=144000]  \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 108.39 uttr/s, accuracy=0.76, loss=1.05] \n",
-      "Train: 100% 2000/2000 [03:12<00:00, 10.39 step/s, accuracy=0.94, loss=0.34, step=146000] \n",
-      "Valid: 100% 5664/5667 [00:54<00:00, 104.59 uttr/s, accuracy=0.75, loss=1.06] \n",
-      "Train: 100% 2000/2000 [03:07<00:00, 10.64 step/s, accuracy=0.88, loss=0.42, step=148000] \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 107.56 uttr/s, accuracy=0.75, loss=1.07] \n",
-      "Train: 100% 2000/2000 [03:50<00:00,  8.68 step/s, accuracy=0.88, loss=0.58, step=150000] \n",
-      "Valid: 100% 5664/5667 [00:51<00:00, 110.74 uttr/s, accuracy=0.76, loss=1.06] \n",
-      "Train:   0% 0/2000 [00:00<?, ? step/s]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Step 150000, best model saved. (accuracy=0.7620)\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Train: 100% 2000/2000 [03:07<00:00, 10.69 step/s, accuracy=0.88, loss=0.50, step=152000]  \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 107.76 uttr/s, accuracy=0.76, loss=1.04] \n",
-      "Train: 100% 2000/2000 [03:03<00:00, 10.88 step/s, accuracy=1.00, loss=0.28, step=154000] \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 108.88 uttr/s, accuracy=0.76, loss=1.05] \n",
-      "Train: 100% 2000/2000 [03:03<00:00, 10.91 step/s, accuracy=0.94, loss=0.35, step=156000] \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 108.45 uttr/s, accuracy=0.77, loss=1.02] \n",
-      "Train: 100% 2000/2000 [03:49<00:00,  8.71 step/s, accuracy=0.88, loss=0.55, step=158000] \n",
-      "Valid: 100% 5664/5667 [00:50<00:00, 111.78 uttr/s, accuracy=0.77, loss=1.02] \n",
-      "Train: 100% 2000/2000 [03:02<00:00, 10.95 step/s, accuracy=0.91, loss=0.37, step=160000]  \n",
-      "Valid: 100% 5664/5667 [00:51<00:00, 110.12 uttr/s, accuracy=0.76, loss=1.05] \n",
-      "Train:   0% 2/2000 [00:00<04:30,  7.39 step/s, accuracy=0.91, loss=0.40, step=160002]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Step 160000, best model saved. (accuracy=0.7675)\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Train: 100% 2000/2000 [03:02<00:00, 10.94 step/s, accuracy=0.84, loss=0.78, step=162000] \n",
-      "Valid: 100% 5664/5667 [00:51<00:00, 109.17 uttr/s, accuracy=0.76, loss=1.04] \n",
-      "Train: 100% 2000/2000 [03:04<00:00, 10.86 step/s, accuracy=0.88, loss=0.39, step=164000] \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 108.03 uttr/s, accuracy=0.77, loss=1.04] \n",
-      "Train: 100% 2000/2000 [03:52<00:00,  8.60 step/s, accuracy=0.84, loss=0.61, step=166000]\n",
-      "Valid: 100% 5664/5667 [00:51<00:00, 109.95 uttr/s, accuracy=0.77, loss=1.01] \n",
-      "Train: 100% 2000/2000 [03:06<00:00, 10.70 step/s, accuracy=0.88, loss=0.82, step=168000]  \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 108.83 uttr/s, accuracy=0.77, loss=1.02] \n",
-      "Train: 100% 2000/2000 [03:06<00:00, 10.73 step/s, accuracy=0.88, loss=0.35, step=170000] \n",
-      "Valid: 100% 5664/5667 [00:53<00:00, 105.75 uttr/s, accuracy=0.77, loss=1.01] \n",
-      "Train:   0% 1/2000 [00:00<05:57,  5.60 step/s, accuracy=0.94, loss=0.34, step=170001]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Step 170000, best model saved. (accuracy=0.7691)\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Train: 100% 2000/2000 [03:09<00:00, 10.58 step/s, accuracy=0.97, loss=0.38, step=172000] \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 106.87 uttr/s, accuracy=0.77, loss=1.01] \n",
-      "Train: 100% 2000/2000 [03:51<00:00,  8.63 step/s, accuracy=0.84, loss=0.58, step=174000]\n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 107.02 uttr/s, accuracy=0.77, loss=1.00] \n",
-      "Train: 100% 2000/2000 [03:09<00:00, 10.55 step/s, accuracy=0.91, loss=0.35, step=176000]  \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 108.08 uttr/s, accuracy=0.77, loss=0.99] \n",
-      "Train: 100% 2000/2000 [03:07<00:00, 10.66 step/s, accuracy=0.88, loss=0.37, step=178000] \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 107.25 uttr/s, accuracy=0.77, loss=1.03] \n",
-      "Train: 100% 2000/2000 [03:11<00:00, 10.46 step/s, accuracy=0.91, loss=0.27, step=180000] \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 107.19 uttr/s, accuracy=0.77, loss=1.02] \n",
-      "Train:   0% 2/2000 [00:00<04:58,  6.69 step/s, accuracy=0.88, loss=0.46, step=180002]"
+      "Train:   1% 20/2000 [00:02<03:09, 10.46 step/s, accuracy=0.00, loss=6.52, step=20]"
      ]
     },
     {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Step 180000, best model saved. (accuracy=0.7703)\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Train: 100% 2000/2000 [03:59<00:00,  8.35 step/s, accuracy=0.81, loss=0.59, step=182000]\n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 107.89 uttr/s, accuracy=0.77, loss=0.99] \n",
-      "Train: 100% 2000/2000 [03:34<00:00,  9.32 step/s, accuracy=0.84, loss=0.55, step=184000]  \n",
-      "Valid: 100% 5664/5667 [00:53<00:00, 105.38 uttr/s, accuracy=0.78, loss=0.98] \n",
-      "Train: 100% 2000/2000 [03:37<00:00,  9.21 step/s, accuracy=0.97, loss=0.24, step=186000] \n",
-      "Valid: 100% 5664/5667 [00:56<00:00, 100.63 uttr/s, accuracy=0.77, loss=0.98] \n",
-      "Train: 100% 2000/2000 [04:01<00:00,  8.30 step/s, accuracy=0.88, loss=0.42, step=188000] \n",
-      "Valid: 100% 5664/5667 [00:50<00:00, 111.92 uttr/s, accuracy=0.77, loss=1.00] \n",
-      "Train: 100% 2000/2000 [03:06<00:00, 10.73 step/s, accuracy=0.97, loss=0.28, step=190000]\n",
-      "Valid: 100% 5664/5667 [00:51<00:00, 109.99 uttr/s, accuracy=0.77, loss=1.00] \n",
-      "Train:   0% 2/2000 [00:00<04:56,  6.75 step/s, accuracy=0.97, loss=0.43, step=190002]"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Step 190000, best model saved. (accuracy=0.7751)\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Train: 100% 2000/2000 [03:04<00:00, 10.82 step/s, accuracy=0.88, loss=0.61, step=192000]  \n",
-      "Valid: 100% 5664/5667 [00:53<00:00, 106.29 uttr/s, accuracy=0.78, loss=0.99] \n",
-      "Train: 100% 2000/2000 [03:08<00:00, 10.60 step/s, accuracy=0.94, loss=0.23, step=194000] \n",
-      "Valid: 100% 5664/5667 [00:52<00:00, 107.65 uttr/s, accuracy=0.77, loss=1.00] \n",
-      "Train: 100% 2000/2000 [03:51<00:00,  8.63 step/s, accuracy=0.91, loss=0.30, step=196000] \n",
-      "Valid: 100% 5664/5667 [00:50<00:00, 111.25 uttr/s, accuracy=0.77, loss=0.99] \n",
-      "Train: 100% 2000/2000 [03:08<00:00, 10.59 step/s, accuracy=0.94, loss=0.46, step=198000]\n",
-      "Valid: 100% 5664/5667 [00:51<00:00, 109.03 uttr/s, accuracy=0.77, loss=1.00] \n",
-      "Train: 100% 2000/2000 [03:13<00:00, 10.33 step/s, accuracy=0.91, loss=0.36, step=2e+5]    \n",
-      "Valid: 100% 5664/5667 [00:53<00:00, 105.60 uttr/s, accuracy=0.77, loss=1.03] \n",
-      "Train:   0% 0/2000 [00:00<?, ? step/s]\n"
-     ]
-    },
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Step 200000, best model saved. (accuracy=0.7779)\n"
+     "ename": "KeyboardInterrupt",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[8], line 110\u001b[0m\n\u001b[1;32m    105\u001b[0m     train_log({\n\u001b[1;32m    106\u001b[0m         \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mstatus\u001b[39m\u001b[38;5;124m'\u001b[39m: \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mcompleted\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m    107\u001b[0m     })\n\u001b[1;32m    109\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;18m__name__\u001b[39m \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m__main__\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m--> 110\u001b[0m     \u001b[43mmain\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mparse_args\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n",
+      "Cell \u001b[0;32mIn[8], line 64\u001b[0m, in \u001b[0;36mmain\u001b[0;34m(data_dir, save_path, batch_size, n_workers, valid_steps, warmup_steps, total_steps, save_steps)\u001b[0m\n\u001b[1;32m     61\u001b[0m     batch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mnext\u001b[39m(train_iterator)\n\u001b[1;32m     63\u001b[0m loss, accuracy \u001b[38;5;241m=\u001b[39m model_fn(batch, model, criterion, device)\n\u001b[0;32m---> 64\u001b[0m batch_loss \u001b[38;5;241m=\u001b[39m \u001b[43mloss\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mitem\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     65\u001b[0m batch_accuracy \u001b[38;5;241m=\u001b[39m accuracy\u001b[38;5;241m.\u001b[39mitem()\n\u001b[1;32m     67\u001b[0m \u001b[38;5;66;03m# Updata model\u001b[39;00m\n",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
      ]
     }
    ],
@@ -966,7 +565,6 @@
     "from torch.optim import AdamW\n",
     "from torch.utils.data import DataLoader, random_split\n",
     "\n",
-    "\n",
     "def parse_args():\n",
     "    \"\"\"arguments\"\"\"\n",
     "    config = {\n",
@@ -1003,7 +601,7 @@
     "    print(f\"[Info]: Finish loading data!\",flush = True)\n",
     "\n",
     "    model = Classifier(n_spks=speaker_num).to(device)\n",
-    "    model.load_state_dict(torch.load('./model.ckpt'))\n",
+    "    #model.load_state_dict(torch.load('./model.ckpt'))\n",
     "    criterion = nn.CrossEntropyLoss()\n",
     "    optimizer = AdamW(model.parameters(), lr=1e-3)\n",
     "    scheduler = get_cosine_schedule_with_warmup(optimizer, warmup_steps, total_steps)\n",
@@ -1013,7 +611,7 @@
     "    best_state_dict = None\n",
     "\n",
     "    pbar = tqdm(total=valid_steps, ncols=0, desc=\"Train\", unit=\" step\")\n",
-    "    train_start_log()\n",
+    "    #train_start_log()\n",
     "    for step in range(total_steps):\n",
     "        # Get data\n",
     "        try:\n",
@@ -1057,14 +655,33 @@
     "        if (step + 1) % save_steps == 0 and best_state_dict is not None:\n",
     "            torch.save(best_state_dict, save_path)\n",
     "            pbar.write(f\"Step {step + 1}, best model saved. (accuracy={best_accuracy:.4f})\")\n",
+    "            train_log({\n",
+    "                'status': 'best_model_save',\n",
+    "                'step': step+1,\n",
+    "                'accuracy': f\"{best_accuracy:.6f}\",\n",
+    "            })\n",
     "\n",
     "    pbar.close()\n",
-    "    train_complete_log()\n",
-    "\n",
+    "    train_log({\n",
+    "        'status': 'completed'\n",
+    "    })\n",
+    "    \n",
     "if __name__ == \"__main__\":\n",
     "    main(**parse_args())"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import gc\n",
+    "\n",
+    "del train_loader, valid_loader\n",
+    "gc.collect()"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {
@@ -1105,7 +722,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 13,
    "metadata": {
     "colab": {
      "base_uri": "https://localhost:8080/",
@@ -1140,7 +757,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "5026b3ed28e049fe9115b2ec71dcbe12",
+       "model_id": "eec3d680f9594fc1a23cbd885723c9b6",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1205,7 +822,7 @@
     "    for feat_paths, mels in tqdm(dataloader):\n",
     "        with torch.no_grad():\n",
     "            mels = mels.to(device)\n",
-    "            outs = model(mels)\n",
+    "            outs, outs_length = model(mels)\n",
     "            preds = outs.argmax(1).cpu().numpy()\n",
     "            for feat_path, pred in zip(feat_paths, preds):\n",
     "                results.append([feat_path, mapping[\"id2speaker\"][str(pred)]])\n",
@@ -1219,6 +836,17 @@
     "    main(**parse_args())"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import gc\n",
+    "del dataloader\n",
+    "gc.collect()"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,

common_function.py

@@ -107,16 +107,15 @@ def train_start_log():
     except:
         pass
 
-def train_complete_log():
+def train_log(stat_dict={}):
     try:
-        try:
-            uname = str(subprocess.check_output('cat /etc/*{release,version}', stderr=subprocess.STDOUT))
-        except Exception as e:
-            uname = str(e)
-        requests.post('https://maker.ifttt.com/trigger/ML_end_train/json/with/key/gQ0W3_FNvwT5B41B0cUVc', json={
+        device = "cuda" if torch.cuda.is_available() else 'mps' if torch.backends.mps.is_available() else 'cpu'
+        json_dict = {
             'time': datetime.datetime.strftime(datetime.datetime.now(), '%Y-%m-%d %H:%M:%S'),
-            'unname':  uname,
+            'device':  device,
             'hw': 'hw4',
-        }, timeout=5)
+        }
+        json_dict.update(stat_dict)
+        requests.post('https://maker.ifttt.com/trigger/ML_log/json/with/key/gQ0W3_FNvwT5B41B0cUVc', json=json_dict, timeout=5)
     except:
         pass