{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "5908e829",
   "metadata": {},
   "source": [
    "# Optimization tutorial"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7e7ea27e-aca8-4bdc-bcdb-dda20d6ac045",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\user\\conda\\envs\\ice_testing\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from ice.remaining_useful_life_estimation.datasets import RulCmapss\n",
    "from ice.remaining_useful_life_estimation.models import MLP"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f9f612b9-f990-40f8-9191-af341827f7d9",
   "metadata": {},
   "source": [
    "Create the MLP model and dataset class."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "81f891c7-de45-448f-a65b-806f2446e6e5",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Reading data/C-MAPSS/fd1_train.csv: 100%|██████████| 20631/20631 [00:00<00:00, 517496.66it/s]\n",
      "Reading data/C-MAPSS/fd2_train.csv: 100%|██████████| 53759/53759 [00:00<00:00, 539376.73it/s]\n",
      "Reading data/C-MAPSS/fd3_train.csv: 100%|██████████| 24720/24720 [00:00<00:00, 506176.62it/s]\n",
      "Reading data/C-MAPSS/fd4_train.csv: 100%|██████████| 61249/61249 [00:00<00:00, 539069.75it/s]\n",
      "Reading data/C-MAPSS/fd1_train.csv: 100%|██████████| 20631/20631 [00:00<00:00, 544734.35it/s]\n",
      "Reading data/C-MAPSS/fd2_train.csv: 100%|██████████| 53759/53759 [00:00<00:00, 481589.37it/s]\n",
      "Reading data/C-MAPSS/fd3_train.csv: 100%|██████████| 24720/24720 [00:00<00:00, 427933.68it/s]\n",
      "Reading data/C-MAPSS/fd4_train.csv: 100%|██████████| 61249/61249 [00:00<00:00, 546792.80it/s]\n",
      "Reading data/C-MAPSS/fd1_test.csv: 100%|██████████| 13097/13097 [00:00<00:00, 505412.69it/s]\n",
      "Reading data/C-MAPSS/fd2_test.csv: 100%|██████████| 33991/33991 [00:00<00:00, 478933.98it/s]\n",
      "Reading data/C-MAPSS/fd3_test.csv: 100%|██████████| 16598/16598 [00:00<00:00, 520419.66it/s]\n",
      "Reading data/C-MAPSS/fd4_test.csv: 100%|██████████| 41214/41214 [00:00<00:00, 537036.66it/s]\n",
      "Reading data/C-MAPSS/fd1_test.csv: 100%|██████████| 13097/13097 [00:00<00:00, 515225.24it/s]\n",
      "Reading data/C-MAPSS/fd2_test.csv: 100%|██████████| 33991/33991 [00:00<00:00, 520644.44it/s]\n",
      "Reading data/C-MAPSS/fd3_test.csv: 100%|██████████| 16598/16598 [00:00<00:00, 489809.10it/s]\n",
      "Reading data/C-MAPSS/fd4_test.csv: 100%|██████████| 41214/41214 [00:00<00:00, 537023.31it/s]\n"
     ]
    }
   ],
   "source": [
    "dataset_class = RulCmapss()\n",
    "data, target = dataset_class.df[0], dataset_class.target[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6fc9029e-7ce9-4cd8-9353-087b66556b17",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = MLP(device=\"cuda\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f8ff1a72-19f4-4389-9362-35a80b8fb851",
   "metadata": {},
   "source": [
    "Optimization **without changing the complexity** of the training process. Tune the lr of the training procedure using validation loss as optimization target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c9e23868-09cb-4693-ad67-f1104245b7df",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-08-13 09:53:33,784] A new study created in memory with name: /parameter_lr study\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trial step with lr = 0.00018951382914416393\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Creating sequence of samples: 100%|██████████| 100/100 [00:00<00:00, 33442.07it/s]\n",
      "Epochs ...:   0%|          | 0/5 [00:00<?, ?it/s]\n",
      "Steps ...:   0%|          | 0/232 [00:00<?, ?it/s]\u001b[A\n",
      "Steps ...:  10%|█         | 24/232 [00:00<00:00, 237.23it/s]\u001b[A\n",
      "Steps ...:  43%|████▎     | 99/232 [00:00<00:00, 535.34it/s]\u001b[A\n",
      "Steps ...:  75%|███████▌  | 174/232 [00:00<00:00, 631.00it/s]\u001b[A\n",
      "Epochs ...:  20%|██        | 1/5 [00:00<00:01,  2.31it/s]    \u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1, Loss: 39.3309\n",
      "Epoch 1, Validation Loss: 39.3238, Metrics: {'rmse': 50.063830627114406, 'cmapss_score': 1009001.2641988464}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Steps ...:   0%|          | 0/232 [00:00<?, ?it/s]\u001b[A\n",
      "Steps ...:  30%|██▉       | 69/232 [00:00<00:00, 686.55it/s]\u001b[A\n",
      "Steps ...:  62%|██████▏   | 143/232 [00:00<00:00, 715.14it/s]\u001b[A\n",
      "Steps ...:  94%|█████████▍| 218/232 [00:00<00:00, 730.32it/s]\u001b[A\n",
      "Epochs ...:  40%|████      | 2/5 [00:00<00:01,  2.54it/s]    \u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2, Loss: 36.8534\n",
      "Epoch 2, Validation Loss: 32.2449, Metrics: {'rmse': 40.68462185124231, 'cmapss_score': 326161.11634528585}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Steps ...:   0%|          | 0/232 [00:00<?, ?it/s]\u001b[A\n",
      "Steps ...:  33%|███▎      | 76/232 [00:00<00:00, 751.21it/s]\u001b[A\n",
      "Steps ...:  66%|██████▌   | 152/232 [00:00<00:00, 755.41it/s]\u001b[A\n",
      "Steps ...:  98%|█████████▊| 228/232 [00:00<00:00, 756.70it/s]\u001b[A\n",
      "Epochs ...:  60%|██████    | 3/5 [00:01<00:00,  2.68it/s]    \u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3, Loss: 22.9493\n",
      "Epoch 3, Validation Loss: 29.7487, Metrics: {'rmse': 38.11920046818511, 'cmapss_score': 251687.82706875}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Steps ...:   0%|          | 0/232 [00:00<?, ?it/s]\u001b[A\n",
      "Steps ...:  32%|███▏      | 75/232 [00:00<00:00, 745.06it/s]\u001b[A\n",
      "Steps ...:  65%|██████▌   | 151/232 [00:00<00:00, 750.90it/s]\u001b[A\n",
      "Steps ...:  98%|█████████▊| 227/232 [00:00<00:00, 753.53it/s]\u001b[A\n",
      "Epochs ...:  80%|████████  | 4/5 [00:01<00:00,  2.74it/s]    \u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 4, Loss: 27.0963\n",
      "Epoch 4, Validation Loss: 28.3393, Metrics: {'rmse': 36.542765927986274, 'cmapss_score': 207206.28175345066}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Steps ...:   0%|          | 0/232 [00:00<?, ?it/s]\u001b[A\n",
      "Steps ...:  32%|███▏      | 74/232 [00:00<00:00, 735.12it/s]\u001b[A\n",
      "Steps ...:  65%|██████▌   | 151/232 [00:00<00:00, 752.66it/s]\u001b[A\n",
      "Steps ...:  98%|█████████▊| 227/232 [00:00<00:00, 755.28it/s]\u001b[A\n",
      "Epochs ...: 100%|██████████| 5/5 [00:01<00:00,  2.70it/s]    \u001b[A\n",
      "[I 2024-08-13 09:53:36,351] Trial 0 finished with value: 26.89373407131288 and parameters: {'lr': 0.00018951382914416393}. Best is trial 0 with value: 26.89373407131288.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 5, Loss: 30.0107\n",
      "Epoch 5, Validation Loss: 26.8937, Metrics: {'rmse': 34.96109612817986, 'cmapss_score': 171362.21049284632}\n",
      "trial step with lr = 0.0007874022874638446\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Creating sequence of samples: 100%|██████████| 100/100 [00:00<00:00, 12542.02it/s]\n",
      "Epochs ...:   0%|          | 0/5 [00:00<?, ?it/s]\n",
      "Steps ...:   0%|          | 0/232 [00:00<?, ?it/s]\u001b[A\n",
      "Steps ...:  17%|█▋        | 39/232 [00:00<00:00, 389.34it/s]\u001b[A\n",
      "Steps ...:  48%|████▊     | 111/232 [00:00<00:00, 581.84it/s]\u001b[A\n",
      "Steps ...:  81%|████████  | 187/232 [00:00<00:00, 661.02it/s]\u001b[A\n",
      "Epochs ...:  20%|██        | 1/5 [00:00<00:01,  2.48it/s]    \u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1, Loss: 32.6104\n",
      "Epoch 1, Validation Loss: 28.4569, Metrics: {'rmse': 36.641393207296055, 'cmapss_score': 206445.81209835963}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Steps ...:   0%|          | 0/232 [00:00<?, ?it/s]\u001b[A\n",
      "Steps ...:  33%|███▎      | 76/232 [00:00<00:00, 755.00it/s]\u001b[A\n",
      "Steps ...:  66%|██████▌   | 153/232 [00:00<00:00, 760.84it/s]\u001b[A\n",
      "Steps ...:  99%|█████████▉| 230/232 [00:00<00:00, 752.50it/s]\u001b[A\n",
      "Epochs ...:  40%|████      | 2/5 [00:00<00:01,  2.69it/s]    \u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2, Loss: 29.5834\n",
      "Epoch 2, Validation Loss: 22.3682, Metrics: {'rmse': 29.68363304759782, 'cmapss_score': 98649.45005642212}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Steps ...:   0%|          | 0/232 [00:00<?, ?it/s]\u001b[A\n",
      "Steps ...:  33%|███▎      | 77/232 [00:00<00:00, 764.93it/s]\u001b[A\n",
      "Steps ...:  66%|██████▋   | 154/232 [00:00<00:00, 498.98it/s]\u001b[A\n",
      "Steps ...:  91%|█████████ | 210/232 [00:00<00:00, 435.48it/s]\u001b[A\n",
      "Epochs ...:  60%|██████    | 3/5 [00:01<00:00,  2.10it/s]    \u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3, Loss: 17.5193\n",
      "Epoch 3, Validation Loss: 17.9444, Metrics: {'rmse': 24.369740561776197, 'cmapss_score': 82673.16229614464}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Steps ...:   0%|          | 0/232 [00:00<?, ?it/s]\u001b[A\n",
      "Steps ...:  16%|█▌        | 37/232 [00:00<00:00, 363.96it/s]\u001b[A\n",
      "Steps ...:  32%|███▏      | 74/232 [00:00<00:00, 359.81it/s]\u001b[A\n",
      "Steps ...:  47%|████▋     | 110/232 [00:00<00:00, 355.65it/s]\u001b[A\n",
      "Steps ...:  63%|██████▎   | 146/232 [00:00<00:00, 352.98it/s]\u001b[A\n",
      "Steps ...:  79%|███████▉  | 183/232 [00:00<00:00, 357.91it/s]\u001b[A\n",
      "Steps ...:  94%|█████████▍| 219/232 [00:00<00:00, 355.44it/s]\u001b[A\n",
      "Epochs ...:  80%|████████  | 4/5 [00:02<00:00,  1.72it/s]    \u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 4, Loss: 16.3577\n",
      "Epoch 4, Validation Loss: 16.1891, Metrics: {'rmse': 22.407311130072234, 'cmapss_score': 90375.15431472883}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Steps ...:   0%|          | 0/232 [00:00<?, ?it/s]\u001b[A\n",
      "Steps ...:  16%|█▌        | 36/232 [00:00<00:00, 357.63it/s]\u001b[A\n",
      "Steps ...:  31%|███       | 72/232 [00:00<00:00, 343.62it/s]\u001b[A\n",
      "Steps ...:  47%|████▋     | 108/232 [00:00<00:00, 346.81it/s]\u001b[A\n",
      "Steps ...:  63%|██████▎   | 146/232 [00:00<00:00, 356.10it/s]\u001b[A\n",
      "Steps ...:  79%|███████▉  | 183/232 [00:00<00:00, 360.17it/s]\u001b[A\n",
      "Steps ...:  95%|█████████▍| 220/232 [00:00<00:00, 358.45it/s]\u001b[A\n",
      "Epochs ...: 100%|██████████| 5/5 [00:02<00:00,  1.76it/s]    \u001b[A\n",
      "[I 2024-08-13 09:53:39,216] Trial 1 finished with value: 16.189056094099836 and parameters: {'lr': 0.0007874022874638446}. Best is trial 1 with value: 16.189056094099836.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 5, Loss: 18.7968\n",
      "Epoch 5, Validation Loss: 16.4563, Metrics: {'rmse': 22.47353109963666, 'cmapss_score': 73324.81707642713}\n",
      "trial step with lr = 0.00012086132027556038\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Creating sequence of samples: 100%|██████████| 100/100 [00:00<00:00, 12541.65it/s]\n",
      "Epochs ...:   0%|          | 0/5 [00:00<?, ?it/s]\n",
      "Steps ...:   0%|          | 0/232 [00:00<?, ?it/s]\u001b[A\n",
      "Steps ...:  15%|█▍        | 34/232 [00:00<00:00, 337.76it/s]\u001b[A\n",
      "Steps ...:  31%|███       | 71/232 [00:00<00:00, 351.34it/s]\u001b[A\n",
      "Steps ...:  46%|████▌     | 107/232 [00:00<00:00, 354.90it/s]\u001b[A\n",
      "Steps ...:  62%|██████▏   | 143/232 [00:00<00:00, 356.58it/s]\u001b[A\n",
      "Steps ...:  78%|███████▊  | 180/232 [00:00<00:00, 357.98it/s]\u001b[A\n",
      "Steps ...:  94%|█████████▎| 217/232 [00:00<00:00, 358.82it/s]\u001b[A\n",
      "Epochs ...:  20%|██        | 1/5 [00:00<00:02,  1.35it/s]    \u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1, Loss: 42.6350\n",
      "Epoch 1, Validation Loss: 41.5396, Metrics: {'rmse': 53.309621676839306, 'cmapss_score': 1652572.5547515503}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Steps ...:   0%|          | 0/232 [00:00<?, ?it/s]\u001b[A\n",
      "Steps ...:  16%|█▌        | 36/232 [00:00<00:00, 355.25it/s]\u001b[A\n",
      "Steps ...:  31%|███       | 72/232 [00:00<00:00, 356.65it/s]\u001b[A\n",
      "Steps ...:  47%|████▋     | 108/232 [00:00<00:00, 357.09it/s]\u001b[A\n",
      "Steps ...:  62%|██████▏   | 144/232 [00:00<00:00, 355.91it/s]\u001b[A\n",
      "Steps ...:  78%|███████▊  | 181/232 [00:00<00:00, 360.10it/s]\u001b[A\n",
      "Steps ...:  94%|█████████▍| 218/232 [00:00<00:00, 359.01it/s]\u001b[A\n",
      "Epochs ...:  40%|████      | 2/5 [00:01<00:02,  1.35it/s]    \u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2, Loss: 36.4397\n",
      "Epoch 2, Validation Loss: 37.1115, Metrics: {'rmse': 46.95681782704423, 'cmapss_score': 653010.5664245718}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Steps ...:   0%|          | 0/232 [00:00<?, ?it/s]\u001b[A\n",
      "Steps ...:  16%|█▌        | 36/232 [00:00<00:00, 351.65it/s]\u001b[A\n",
      "Steps ...:  31%|███       | 72/232 [00:00<00:00, 353.10it/s]\u001b[A\n",
      "Steps ...:  47%|████▋     | 109/232 [00:00<00:00, 356.46it/s]\u001b[A\n",
      "Steps ...:  62%|██████▎   | 145/232 [00:00<00:00, 356.14it/s]\u001b[A\n",
      "Steps ...:  78%|███████▊  | 181/232 [00:00<00:00, 355.91it/s]\u001b[A\n",
      "Steps ...:  94%|█████████▎| 217/232 [00:00<00:00, 354.12it/s]\u001b[A\n",
      "Epochs ...:  60%|██████    | 3/5 [00:02<00:01,  1.34it/s]    \u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3, Loss: 25.7715\n",
      "Epoch 3, Validation Loss: 32.9065, Metrics: {'rmse': 41.430416515203575, 'cmapss_score': 349818.50039921864}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Steps ...:   0%|          | 0/232 [00:00<?, ?it/s]\u001b[A\n",
      "Steps ...:  16%|█▌        | 37/232 [00:00<00:00, 360.43it/s]\u001b[A\n",
      "Steps ...:  32%|███▏      | 74/232 [00:00<00:00, 359.15it/s]\u001b[A\n",
      "Steps ...:  48%|████▊     | 111/232 [00:00<00:00, 359.73it/s]\u001b[A\n",
      "Steps ...:  64%|██████▍   | 148/232 [00:00<00:00, 360.01it/s]\u001b[A\n",
      "Steps ...:  80%|███████▉  | 185/232 [00:00<00:00, 359.48it/s]\u001b[A\n",
      "Steps ...:  95%|█████████▌| 221/232 [00:00<00:00, 356.30it/s]\u001b[A\n",
      "Epochs ...:  80%|████████  | 4/5 [00:02<00:00,  1.35it/s]    \u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 4, Loss: 29.5199\n",
      "Epoch 4, Validation Loss: 30.6848, Metrics: {'rmse': 39.035223216989905, 'cmapss_score': 274871.78833234054}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Steps ...:   0%|          | 0/232 [00:00<?, ?it/s]\u001b[A\n",
      "Steps ...:  16%|█▌        | 36/232 [00:00<00:00, 350.68it/s]\u001b[A\n",
      "Steps ...:  31%|███       | 72/232 [00:00<00:00, 352.70it/s]\u001b[A\n",
      "Steps ...:  47%|████▋     | 108/232 [00:00<00:00, 348.67it/s]\u001b[A\n",
      "Steps ...:  62%|██████▏   | 143/232 [00:00<00:00, 332.92it/s]\u001b[A\n",
      "Steps ...:  77%|███████▋  | 178/232 [00:00<00:00, 338.04it/s]\u001b[A\n",
      "Steps ...:  92%|█████████▏| 214/232 [00:00<00:00, 343.43it/s]\u001b[A\n",
      "Epochs ...: 100%|██████████| 5/5 [00:03<00:00,  1.34it/s]    \u001b[A\n",
      "[I 2024-08-13 09:53:42,980] Trial 2 finished with value: 29.60851450664241 and parameters: {'lr': 0.00012086132027556038}. Best is trial 1 with value: 16.189056094099836.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 5, Loss: 33.5629\n",
      "Epoch 5, Validation Loss: 29.6085, Metrics: {'rmse': 37.93360429205696, 'cmapss_score': 242353.786303807}\n",
      "Best hyperparameters: {'lr': 0.0007874022874638446}\n",
      "Best trial: FrozenTrial(number=1, state=1, values=[16.189056094099836], datetime_start=datetime.datetime(2024, 8, 13, 9, 53, 36, 351082), datetime_complete=datetime.datetime(2024, 8, 13, 9, 53, 39, 216093), params={'lr': 0.0007874022874638446}, user_attrs={}, system_attrs={}, intermediate_values={1: 28.456903271558808, 2: 22.368176297443668, 3: 17.944359942180355, 4: 16.189056094099836, 5: 16.456260332247105}, distributions={'lr': FloatDistribution(high=0.001, log=False, low=5e-05, step=None)}, trial_id=1, value=None)\n"
     ]
    }
   ],
   "source": [
    "# model_class.optimize(data, target, optimize_parameter, optimize_range, direction, n_trials, epochs, optimize_metric)\n",
    "model.optimize(data, target, optimize_parameter=\"lr\", optimize_range=(5e-5, 1e-3), direction=\"minimize\", n_trials=3, epochs=5) # if optimize_metric is None, than validation loss is using as optimization target"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "40c89851-8388-4444-8b2c-e22b9a2c0bf4",
   "metadata": {},
   "source": [
    "Optimization **with changing the complexity** of the training process. Tune the MLP hidden dimension size using MSE metric as optimization target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6acb85da-401d-447a-8fe0-00fefb7a78f9",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-08-13 09:53:42,992] A new study created in memory with name: /parameter_hidden_dim study\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trial step with hidden_dim = 702\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Creating sequence of samples: 100%|██████████| 100/100 [00:00<00:00, 11148.24it/s]\n",
      "Epochs ...:   0%|          | 0/5 [00:00<?, ?it/s]\n",
      "Steps ...:   0%|          | 0/232 [00:00<?, ?it/s]\u001b[A\n",
      "Steps ...:  14%|█▍        | 32/232 [00:00<00:00, 315.23it/s]\u001b[A\n",
      "Steps ...:  29%|██▉       | 67/232 [00:00<00:00, 333.10it/s]\u001b[A\n",
      "Steps ...:  44%|████▍     | 102/232 [00:00<00:00, 338.76it/s]\u001b[A\n",
      "Steps ...:  59%|█████▉    | 137/232 [00:00<00:00, 339.63it/s]\u001b[A\n",
      "Steps ...:  74%|███████▍  | 172/232 [00:00<00:00, 340.10it/s]\u001b[A\n",
      "Steps ...:  89%|████████▉ | 207/232 [00:00<00:00, 341.73it/s]\u001b[A\n",
      "Epochs ...:  20%|██        | 1/5 [00:00<00:03,  1.28it/s]    \u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1, Loss: 41.7009\n",
      "Epoch 1, Validation Loss: 41.0446, Metrics: {'rmse': 52.46324288978529, 'cmapss_score': 1459642.2102222845}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Steps ...:   0%|          | 0/232 [00:00<?, ?it/s]\u001b[A\n",
      "Steps ...:  16%|█▌        | 36/232 [00:00<00:00, 350.68it/s]\u001b[A\n",
      "Steps ...:  31%|███       | 72/232 [00:00<00:00, 354.74it/s]\u001b[A\n",
      "Steps ...:  47%|████▋     | 108/232 [00:00<00:00, 354.46it/s]\u001b[A\n",
      "Steps ...:  62%|██████▏   | 144/232 [00:00<00:00, 354.32it/s]\u001b[A\n",
      "Steps ...:  78%|███████▊  | 181/232 [00:00<00:00, 357.14it/s]\u001b[A\n",
      "Steps ...:  94%|█████████▎| 217/232 [00:00<00:00, 354.29it/s]\u001b[A\n",
      "Epochs ...:  40%|████      | 2/5 [00:01<00:02,  1.31it/s]    \u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2, Loss: 34.5571\n",
      "Epoch 2, Validation Loss: 35.5294, Metrics: {'rmse': 44.69688917194367, 'cmapss_score': 487095.93532787054}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Steps ...:   0%|          | 0/232 [00:00<?, ?it/s]\u001b[A\n",
      "Steps ...:  14%|█▍        | 33/232 [00:00<00:00, 328.99it/s]\u001b[A\n",
      "Steps ...:  29%|██▉       | 67/232 [00:00<00:00, 335.00it/s]\u001b[A\n",
      "Steps ...:  44%|████▍     | 103/232 [00:00<00:00, 342.81it/s]\u001b[A\n",
      "Steps ...:  59%|█████▉    | 138/232 [00:00<00:00, 342.07it/s]\u001b[A\n",
      "Steps ...:  75%|███████▍  | 173/232 [00:00<00:00, 340.46it/s]\u001b[A\n",
      "Steps ...:  90%|████████▉ | 208/232 [00:00<00:00, 342.89it/s]\u001b[A\n",
      "Epochs ...:  60%|██████    | 3/5 [00:02<00:01,  1.30it/s]    \u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3, Loss: 24.0595\n",
      "Epoch 3, Validation Loss: 31.7725, Metrics: {'rmse': 40.124605880566875, 'cmapss_score': 305081.3116466362}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Steps ...:   0%|          | 0/232 [00:00<?, ?it/s]\u001b[A\n",
      "Steps ...:  15%|█▌        | 35/232 [00:00<00:00, 344.29it/s]\u001b[A\n",
      "Steps ...:  30%|███       | 70/232 [00:00<00:00, 344.29it/s]\u001b[A\n",
      "Steps ...:  46%|████▌     | 106/232 [00:00<00:00, 347.96it/s]\u001b[A\n",
      "Steps ...:  61%|██████    | 142/232 [00:00<00:00, 350.98it/s]\u001b[A\n",
      "Steps ...:  77%|███████▋  | 178/232 [00:00<00:00, 350.24it/s]\u001b[A\n",
      "Steps ...:  92%|█████████▏| 214/232 [00:00<00:00, 349.78it/s]\u001b[A\n",
      "Epochs ...:  80%|████████  | 4/5 [00:03<00:00,  1.30it/s]    \u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 4, Loss: 27.7565\n",
      "Epoch 4, Validation Loss: 29.9569, Metrics: {'rmse': 38.28838337636238, 'cmapss_score': 252440.2810893617}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Steps ...:   0%|          | 0/232 [00:00<?, ?it/s]\u001b[A\n",
      "Steps ...:  14%|█▍        | 32/232 [00:00<00:00, 314.78it/s]\u001b[A\n",
      "Steps ...:  28%|██▊       | 66/232 [00:00<00:00, 324.48it/s]\u001b[A\n",
      "Steps ...:  44%|████▍     | 102/232 [00:00<00:00, 337.97it/s]\u001b[A\n",
      "Steps ...:  59%|█████▊    | 136/232 [00:00<00:00, 338.51it/s]\u001b[A\n",
      "Steps ...:  73%|███████▎  | 170/232 [00:00<00:00, 330.41it/s]\u001b[A\n",
      "Steps ...:  88%|████████▊ | 204/232 [00:00<00:00, 324.28it/s]\u001b[A\n",
      "Epochs ...: 100%|██████████| 5/5 [00:03<00:00,  1.29it/s]    \u001b[A\n",
      "[I 2024-08-13 09:53:46,884] Trial 0 finished with value: 37.12774637917532 and parameters: {'hidden_dim': 702}. Best is trial 0 with value: 37.12774637917532.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 5, Loss: 31.4343\n",
      "Epoch 5, Validation Loss: 28.8321, Metrics: {'rmse': 37.12774637917532, 'cmapss_score': 219477.3266793877}\n",
      "trial step with hidden_dim = 662\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Creating sequence of samples: 100%|██████████| 100/100 [00:00<00:00, 12542.40it/s]\n",
      "Epochs ...:   0%|          | 0/5 [00:00<?, ?it/s]\n",
      "Steps ...:   0%|          | 0/232 [00:00<?, ?it/s]\u001b[A\n",
      "Steps ...:  15%|█▍        | 34/232 [00:00<00:00, 334.45it/s]\u001b[A\n",
      "Steps ...:  30%|██▉       | 69/232 [00:00<00:00, 342.21it/s]\u001b[A\n",
      "Steps ...:  45%|████▍     | 104/232 [00:00<00:00, 337.83it/s]\u001b[A\n",
      "Steps ...:  60%|█████▉    | 139/232 [00:00<00:00, 340.97it/s]\u001b[A\n",
      "Steps ...:  75%|███████▌  | 174/232 [00:00<00:00, 342.74it/s]\u001b[A\n",
      "Steps ...:  90%|█████████ | 209/232 [00:00<00:00, 344.41it/s]\u001b[A\n",
      "Epochs ...:  20%|██        | 1/5 [00:00<00:03,  1.26it/s]    \u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1, Loss: 41.5648\n",
      "Epoch 1, Validation Loss: 41.1247, Metrics: {'rmse': 52.64622625632633, 'cmapss_score': 1497278.352392366}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Steps ...:   0%|          | 0/232 [00:00<?, ?it/s]\u001b[A\n",
      "Steps ...:  14%|█▍        | 32/232 [00:00<00:00, 319.07it/s]\u001b[A\n",
      "Steps ...:  28%|██▊       | 66/232 [00:00<00:00, 330.08it/s]\u001b[A\n",
      "Steps ...:  44%|████▎     | 101/232 [00:00<00:00, 338.13it/s]\u001b[A\n",
      "Steps ...:  59%|█████▊    | 136/232 [00:00<00:00, 341.91it/s]\u001b[A\n",
      "Steps ...:  74%|███████▍  | 172/232 [00:00<00:00, 346.34it/s]\u001b[A\n",
      "Steps ...:  89%|████████▉ | 207/232 [00:00<00:00, 346.80it/s]\u001b[A\n",
      "Epochs ...:  40%|████      | 2/5 [00:01<00:02,  1.27it/s]    \u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2, Loss: 37.5466\n",
      "Epoch 2, Validation Loss: 35.6801, Metrics: {'rmse': 45.00773698916855, 'cmapss_score': 511059.8643841665}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Steps ...:   0%|          | 0/232 [00:00<?, ?it/s]\u001b[A\n",
      "Steps ...:  14%|█▍        | 33/232 [00:00<00:00, 321.46it/s]\u001b[A\n",
      "Steps ...:  28%|██▊       | 66/232 [00:00<00:00, 320.47it/s]\u001b[A\n",
      "Steps ...:  43%|████▎     | 99/232 [00:00<00:00, 315.95it/s]\u001b[A\n",
      "Steps ...:  57%|█████▋    | 132/232 [00:00<00:00, 318.65it/s]\u001b[A\n",
      "Steps ...:  71%|███████   | 165/232 [00:00<00:00, 321.90it/s]\u001b[A\n",
      "Steps ...:  86%|████████▌ | 200/232 [00:00<00:00, 330.28it/s]\u001b[A\n",
      "Epochs ...:  60%|██████    | 3/5 [00:02<00:01,  1.25it/s]    \u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3, Loss: 26.5062\n",
      "Epoch 3, Validation Loss: 31.7830, Metrics: {'rmse': 40.07700663323968, 'cmapss_score': 305080.57834935177}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Steps ...:   0%|          | 0/232 [00:00<?, ?it/s]\u001b[A\n",
      "Steps ...:  14%|█▍        | 32/232 [00:00<00:00, 318.04it/s]\u001b[A\n",
      "Steps ...:  28%|██▊       | 65/232 [00:00<00:00, 323.80it/s]\u001b[A\n",
      "Steps ...:  44%|████▎     | 101/232 [00:00<00:00, 337.28it/s]\u001b[A\n",
      "Steps ...:  59%|█████▊    | 136/232 [00:00<00:00, 341.38it/s]\u001b[A\n",
      "Steps ...:  74%|███████▎  | 171/232 [00:00<00:00, 342.43it/s]\u001b[A\n",
      "Steps ...:  89%|████████▉ | 207/232 [00:00<00:00, 345.26it/s]\u001b[A\n",
      "Epochs ...:  80%|████████  | 4/5 [00:03<00:00,  1.26it/s]    \u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 4, Loss: 28.9049\n",
      "Epoch 4, Validation Loss: 29.9922, Metrics: {'rmse': 38.29525364434387, 'cmapss_score': 252989.93131717102}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Steps ...:   0%|          | 0/232 [00:00<?, ?it/s]\u001b[A\n",
      "Steps ...:  15%|█▌        | 35/232 [00:00<00:00, 345.28it/s]\u001b[A\n",
      "Steps ...:  31%|███       | 71/232 [00:00<00:00, 350.49it/s]\u001b[A\n",
      "Steps ...:  46%|████▌     | 107/232 [00:00<00:00, 352.15it/s]\u001b[A\n",
      "Steps ...:  62%|██████▏   | 143/232 [00:00<00:00, 349.53it/s]\u001b[A\n",
      "Steps ...:  77%|███████▋  | 179/232 [00:00<00:00, 351.74it/s]\u001b[A\n",
      "Steps ...:  93%|█████████▎| 215/232 [00:00<00:00, 348.30it/s]\u001b[A\n",
      "Epochs ...: 100%|██████████| 5/5 [00:03<00:00,  1.27it/s]    \u001b[A\n",
      "[I 2024-08-13 09:53:50,841] Trial 1 finished with value: 37.207252080735245 and parameters: {'hidden_dim': 662}. Best is trial 0 with value: 37.12774637917532.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 5, Loss: 31.7666\n",
      "Epoch 5, Validation Loss: 28.9132, Metrics: {'rmse': 37.207252080735245, 'cmapss_score': 221679.17640028123}\n",
      "trial step with hidden_dim = 879\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Creating sequence of samples: 100%|██████████| 100/100 [00:00<00:00, 11148.54it/s]\n",
      "Epochs ...:   0%|          | 0/5 [00:00<?, ?it/s]\n",
      "Steps ...:   0%|          | 0/232 [00:00<?, ?it/s]\u001b[A\n",
      "Steps ...:  15%|█▌        | 35/232 [00:00<00:00, 340.94it/s]\u001b[A\n",
      "Steps ...:  31%|███       | 71/232 [00:00<00:00, 348.66it/s]\u001b[A\n",
      "Steps ...:  46%|████▌     | 107/232 [00:00<00:00, 351.77it/s]\u001b[A\n",
      "Steps ...:  62%|██████▏   | 143/232 [00:00<00:00, 354.07it/s]\u001b[A\n",
      "Steps ...:  77%|███████▋  | 179/232 [00:00<00:00, 351.60it/s]\u001b[A\n",
      "Steps ...:  93%|█████████▎| 215/232 [00:00<00:00, 353.62it/s]\u001b[A\n",
      "Epochs ...:  20%|██        | 1/5 [00:00<00:03,  1.32it/s]    \u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1, Loss: 40.2073\n",
      "Epoch 1, Validation Loss: 40.6210, Metrics: {'rmse': 51.9431875404865, 'cmapss_score': 1331357.9658768093}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Steps ...:   0%|          | 0/232 [00:00<?, ?it/s]\u001b[A\n",
      "Steps ...:  16%|█▌        | 36/232 [00:00<00:00, 350.68it/s]\u001b[A\n",
      "Steps ...:  31%|███       | 72/232 [00:00<00:00, 348.69it/s]\u001b[A\n",
      "Steps ...:  47%|████▋     | 108/232 [00:00<00:00, 349.60it/s]\u001b[A\n",
      "Steps ...:  62%|██████▎   | 145/232 [00:00<00:00, 354.45it/s]\u001b[A\n",
      "Steps ...:  78%|███████▊  | 181/232 [00:00<00:00, 353.68it/s]\u001b[A\n",
      "Steps ...:  94%|█████████▎| 217/232 [00:00<00:00, 352.66it/s]\u001b[A\n",
      "Epochs ...:  40%|████      | 2/5 [00:01<00:02,  1.32it/s]    \u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2, Loss: 37.4625\n",
      "Epoch 2, Validation Loss: 34.5596, Metrics: {'rmse': 43.41147259207038, 'cmapss_score': 421283.66798380984}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Steps ...:   0%|          | 0/232 [00:00<?, ?it/s]\u001b[A\n",
      "Steps ...:  16%|█▌        | 36/232 [00:00<00:00, 358.52it/s]\u001b[A\n",
      "Steps ...:  31%|███       | 72/232 [00:00<00:00, 356.73it/s]\u001b[A\n",
      "Steps ...:  47%|████▋     | 109/232 [00:00<00:00, 358.43it/s]\u001b[A\n",
      "Steps ...:  62%|██████▎   | 145/232 [00:00<00:00, 355.35it/s]\u001b[A\n",
      "Steps ...:  78%|███████▊  | 182/232 [00:00<00:00, 358.45it/s]\u001b[A\n",
      "Steps ...:  94%|█████████▍| 218/232 [00:00<00:00, 356.98it/s]\u001b[A\n",
      "Epochs ...:  60%|██████    | 3/5 [00:02<00:01,  1.33it/s]    \u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3, Loss: 23.8398\n",
      "Epoch 3, Validation Loss: 30.9023, Metrics: {'rmse': 39.31126980464057, 'cmapss_score': 286766.446915035}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Steps ...:   0%|          | 0/232 [00:00<?, ?it/s]\u001b[A\n",
      "Steps ...:  16%|█▌        | 36/232 [00:00<00:00, 357.63it/s]\u001b[A\n",
      "Steps ...:  31%|███       | 72/232 [00:00<00:00, 357.63it/s]\u001b[A\n",
      "Steps ...:  47%|████▋     | 109/232 [00:00<00:00, 358.92it/s]\u001b[A\n",
      "Steps ...:  62%|██████▎   | 145/232 [00:00<00:00, 358.41it/s]\u001b[A\n",
      "Steps ...:  78%|███████▊  | 182/232 [00:00<00:00, 359.14it/s]\u001b[A\n",
      "Steps ...:  94%|█████████▍| 218/232 [00:00<00:00, 359.17it/s]\u001b[A\n",
      "Epochs ...:  80%|████████  | 4/5 [00:02<00:00,  1.34it/s]    \u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 4, Loss: 28.2913\n",
      "Epoch 4, Validation Loss: 29.3355, Metrics: {'rmse': 37.68203820452062, 'cmapss_score': 236872.37425031906}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Steps ...:   0%|          | 0/232 [00:00<?, ?it/s]\u001b[A\n",
      "Steps ...:  15%|█▌        | 35/232 [00:00<00:00, 347.69it/s]\u001b[A\n",
      "Steps ...:  30%|███       | 70/232 [00:00<00:00, 347.69it/s]\u001b[A\n",
      "Steps ...:  45%|████▌     | 105/232 [00:00<00:00, 345.31it/s]\u001b[A\n",
      "Steps ...:  60%|██████    | 140/232 [00:00<00:00, 345.55it/s]\u001b[A\n",
      "Steps ...:  76%|███████▌  | 176/232 [00:00<00:00, 348.00it/s]\u001b[A\n",
      "Steps ...:  91%|█████████▏| 212/232 [00:00<00:00, 351.78it/s]\u001b[A\n",
      "Epochs ...: 100%|██████████| 5/5 [00:03<00:00,  1.33it/s]    \u001b[A\n",
      "[I 2024-08-13 09:53:54,617] Trial 2 finished with value: 36.46475764089671 and parameters: {'hidden_dim': 879}. Best is trial 2 with value: 36.46475764089671.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 5, Loss: 31.4287\n",
      "Epoch 5, Validation Loss: 28.2309, Metrics: {'rmse': 36.46475764089671, 'cmapss_score': 203393.37691007284}\n",
      "Best hyperparameters: {'hidden_dim': 879}\n",
      "Best trial: FrozenTrial(number=2, state=1, values=[36.46475764089671], datetime_start=datetime.datetime(2024, 8, 13, 9, 53, 50, 842965), datetime_complete=datetime.datetime(2024, 8, 13, 9, 53, 54, 617905), params={'hidden_dim': 879}, user_attrs={}, system_attrs={}, intermediate_values={1: 40.62102061946218, 2: 34.55964553646925, 3: 30.902342726544635, 4: 29.335512719503264, 5: 28.230895298283276}, distributions={'hidden_dim': IntDistribution(high=1024, log=False, low=256, step=1)}, trial_id=2, value=None)\n"
     ]
    }
   ],
   "source": [
    "# model_class.optimize(data, target, optimize_parameter, optimize_range, direction, n_trials, epochs, optimize_metric)\n",
    "model.optimize(data, target, optimize_parameter=\"hidden_dim\", optimize_range=(256, 1024), direction=\"minimize\", optimize_metric=\"rmse\", n_trials=3, epochs=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "55a39d2e-4684-4869-98ca-337a117dc3df",
   "metadata": {},
   "source": [
    "The best results are printed at the end of optimization processand saved in the *outputs/task_name/traininig/param_name_optimization* folder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "424bd172-58b6-4a12-b379-f2f54fd56364",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}