andras/bipolaroid
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Adjust training script for Residual3

Browse source
This commit is contained in:
Andras Schmelczer 2024-06-22 18:31:43 +01:00
commit e5959268c1
No known key found for this signature in database
GPG key ID: FC8F2C3D3D1A718C
1 changed files with 376 additions and 188 deletions
Download patch file Download diff file Expand all files Collapse all files

564
src/train.ipynb
View file

@ -22,19 +22,10 @@
"import torch\n", "import torch\n",
"import logging\n", "import logging\n",
"import os\n", "import os\n",
"from datetime import datetime\n", "from editor.utils import set_up_logging\n",
"from config import LOGS_PATH\n", "from config import LOGS_PATH\n",
"\n", "\n",
"logging.basicConfig(\n", "set_up_logging(LOGS_PATH)\n",
" level=logging.INFO,\n",
" format=\"%(asctime)s - %(levelname)s - %(message)s\",\n",
" handlers=[\n",
" logging.StreamHandler(),\n",
" logging.FileHandler(\n",
" LOGS_PATH / f\"train-{datetime.now().isoformat(timespec='minutes')}.log\"\n",
" ),\n",
" ],\n",
")\n",
"\n", "\n",
"os.environ[\"PYTORCH_CUDA_ALLOC_CONF\"] = \"expandable_segments:True\"\n", "os.environ[\"PYTORCH_CUDA_ALLOC_CONF\"] = \"expandable_segments:True\"\n",
"device = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device(\"cpu\")\n", "device = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
@ -45,34 +36,49 @@
"cell_type": "code", "cell_type": "code",
"execution_count": 2, "execution_count": 2,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"2024-06-22 15:59:06,999 - INFO - Testing model Dummy\n",
"2024-06-22 15:59:07,004 - INFO - Test passed! Output shape matches input shape.\n",
"2024-06-22 15:59:07,004 - INFO - Testing model SimpleCNN\n",
"2024-06-22 15:59:07,478 - INFO - Test passed! Output shape matches input shape.\n",
"2024-06-22 15:59:07,481 - INFO - Testing model Residual\n",
"2024-06-22 15:59:08,560 - INFO - Test passed! Output shape matches input shape.\n",
"2024-06-22 15:59:08,566 - INFO - Testing model Residual2\n",
"2024-06-22 15:59:09,671 - INFO - Test passed! Output shape matches input shape.\n",
"2024-06-22 15:59:09,676 - INFO - Testing model Residual3\n",
"2024-06-22 15:59:11,272 - INFO - Test passed! Output shape matches input shape.\n"
]
}
],
"source": [ "source": [
"from scipy.stats import loguniform, uniform, randint\n", "from scipy.stats import loguniform, uniform, randint\n",
"from editor.models import MODELS\n", "from editor.models import MODELS, test_models\n",
"\n", "\n",
"common_hyperparameters = {\n", "\n",
" \"batch_size\": [64],\n", "hyperparameters = [{\n",
" \"batch_size\": [32, 64, 128],\n",
" \"edit_count\": [12],\n", " \"edit_count\": [12],\n",
" \"bin_count\": [16],\n", " \"bin_count\": [16],\n",
" \"clip_gradients\": [False],\n", " \"learning_rate\": loguniform(5e-4, 5e-3),\n",
" \"learning_rate\": loguniform(1e-4, 5e-3),\n", " \"scheduler_gamma\": uniform(loc=0.8, scale=0.15),\n",
" \"scheduler_gamma\": uniform(loc=0.7, scale=0.3),\n", " \"num_epochs\": [12],\n",
" \"num_epochs\": [24],\n", " \"elu_alpha\": uniform(0.5, 1.5),\n",
" # \"num_epochs\": randint(5, 10),\n", " \"leaky_relu_slope\": uniform(0, 0.03),\n",
" \"model_type\": list(MODELS.keys()),\n", " \"dropout_prob\": uniform(0, 0.1),\n",
"}\n", " \"features\": [[16, 32, 64], [32, 64, 128], [8, 16, 32], [8, 8, 8], [16, 16, 16]],\n",
"hyperparameters = [\n", " \"kernel_sizes\": [[3, 3, 3]],\n",
" # {\n", " \"model_type\": [\"Residual3\"], # list(MODELS.keys()),\n",
" # **common_hyperparameters,\n", " \"clip_gradients\": [True, False],\n",
" # \"loss\": [\"progressive\"],\n", " \"use_instance_norm\": [True, False],\n",
" # \"loss_sizes\": [[4, 8, 16, 32], [8, 16, 32], [16, 32], [8, 32]],\n", " \"use_elu\": [True, False],\n",
" # \"loss_damping\": uniform(0.2, 5),\n", " \"leaky_relu_alpha\": uniform(0, 0.05),\n",
" # },\n", "}]\n",
" {\n", "\n",
" **common_hyperparameters,\n", "test_models()"
" \"loss\": [\"kl\"],\n",
" },\n",
"]"
] ]
}, },
{ {
@ -111,36 +117,24 @@
"cell_type": "code", "cell_type": "code",
"execution_count": 4, "execution_count": 4,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [],
{
"name": "stdout",
"output_type": "stream",
"text": [
"Testing model Residual\n",
"Test passed! Output shape matches input shape.\n"
]
}
],
"source": [ "source": [
"from typing import Optional\n",
"from torch.utils.tensorboard import SummaryWriter\n", "from torch.utils.tensorboard import SummaryWriter\n",
"from pathlib import Path\n", "from pathlib import Path\n",
"from torch.optim import Adam\n", "from torch.optim import Adam\n",
"from tqdm.notebook import tqdm\n", "from tqdm.notebook import tqdm\n",
"from torch.nn.utils import clip_grad_norm_\n",
"from editor.training import ProgressivePoolingLoss\n",
"from editor.utils import get_next_run_name\n", "from editor.utils import get_next_run_name\n",
"from editor.visualisation import plot_histograms_in_2d\n", "from editor.visualisation import plot_histograms_in_2d\n",
"from editor.models import create_model, test_models\n", "from editor.models import create_model, save_model\n",
"from datetime import timedelta, datetime\n", "from datetime import timedelta, datetime\n",
"import json\n",
"from config import MODELS_PATH, RUNS_PATH, TRAIN_DATA, TEST_DATA\n", "from config import MODELS_PATH, RUNS_PATH, TRAIN_DATA, TEST_DATA\n",
"\n", "\n",
"\n", "\n",
"test_models()\n",
"\n",
"\n",
"def train(\n", "def train(\n",
" hyperparameters: Dict[str, Any], max_duration: timedelta, use_tqdm: bool\n", " hyperparameters: Dict[str, Any],\n",
" max_duration: Optional[timedelta] = None,\n",
" use_tqdm: bool = True,\n",
") -> Path:\n", ") -> Path:\n",
" start_time = datetime.now()\n", " start_time = datetime.now()\n",
"\n", "\n",
@ -149,28 +143,18 @@
" train_data_loader = get_data_loader(TRAIN_DATA, hyperparameters)\n", " train_data_loader = get_data_loader(TRAIN_DATA, hyperparameters)\n",
" test_data_loader = get_data_loader(TEST_DATA, hyperparameters)\n", " test_data_loader = get_data_loader(TEST_DATA, hyperparameters)\n",
"\n", "\n",
" model = (\n", " model = create_model(\n",
" create_model(\n", " type=hyperparameters[\"model_type\"],\n",
" type=hyperparameters[\"model_type\"],\n", " bin_count=hyperparameters[\"bin_count\"],\n",
" bin_count=hyperparameters[\"bin_count\"],\n", " device=device,\n",
" )\n", " ).train()\n",
" .train()\n",
" .to(device)\n",
" )\n",
" writer.add_graph(model, next(iter(train_data_loader))[0].to(device))\n", " writer.add_graph(model, next(iter(train_data_loader))[0].to(device))\n",
"\n", "\n",
" optimizer = Adam(model.parameters(), lr=hyperparameters[\"learning_rate\"])\n", " optimizer = Adam(model.parameters(), lr=hyperparameters[\"learning_rate\"])\n",
" scheduler = torch.optim.lr_scheduler.StepLR(\n", " scheduler = torch.optim.lr_scheduler.StepLR(\n",
" optimizer, step_size=1, gamma=hyperparameters[\"scheduler_gamma\"]\n", " optimizer, step_size=1, gamma=hyperparameters[\"scheduler_gamma\"]\n",
" )\n", " )\n",
"\n", " loss_function = torch.nn.KLDivLoss(reduction=\"batchmean\").to(device)\n",
" loss_function = {\n",
" \"progressive\": lambda: ProgressivePoolingLoss(\n",
" target_sizes=hyperparameters[\"loss_sizes\"],\n",
" damping=hyperparameters[\"loss_damping\"],\n",
" ),\n",
" \"kl\": lambda: torch.nn.KLDivLoss(reduction=\"batchmean\"),\n",
" }[hyperparameters[\"loss\"]]().to(device)\n",
"\n", "\n",
" try:\n", " try:\n",
" for epoch in range(hyperparameters[\"num_epochs\"]):\n", " for epoch in range(hyperparameters[\"num_epochs\"]):\n",
@ -184,30 +168,18 @@
" else train_data_loader\n", " else train_data_loader\n",
" ):\n", " ):\n",
" current_time = datetime.now()\n", " current_time = datetime.now()\n",
" if current_time - start_time > max_duration:\n", " if (\n",
" max_duration is not None\n",
" and current_time - start_time > max_duration\n",
" ):\n",
" raise TimeoutError(f\"Time limit {max_duration} exceeded\")\n", " raise TimeoutError(f\"Time limit {max_duration} exceeded\")\n",
" edited_histogram = edited_histogram.to(device)\n",
" original_histogram = original_histogram.to(device)\n",
"\n", "\n",
" optimizer.zero_grad()\n", " optimizer.zero_grad()\n",
" predicted_original = model(edited_histogram)\n", " predicted_original = model(edited_histogram.to(device))\n",
" sum = torch.sum(predicted_original, dim=(2, 3, 4), keepdim=True)\n", " loss = loss_function(\n",
" predicted_original = predicted_original / sum\n", " torch.log(torch.clamp(predicted_original, 1e-10, 1)),\n",
"\n", " original_histogram.to(device),\n",
" if hyperparameters[\"loss\"] == \"kl\":\n", " )\n",
" predicted_original = torch.clamp(\n",
" predicted_original, 0.0000000000000000000001, 1\n",
" )\n",
"\n",
" loss = {\n",
" \"kl\": lambda: loss_function(\n",
" torch.log(predicted_original),\n",
" original_histogram,\n",
" ),\n",
" \"progressive\": lambda: loss_function(\n",
" predicted_original, original_histogram\n",
" ),\n",
" }[hyperparameters[\"loss\"]]()\n",
"\n", "\n",
" epoch_loss += loss.item()\n", " epoch_loss += loss.item()\n",
" writer.add_scalar(\n", " writer.add_scalar(\n",
@ -216,9 +188,6 @@
" global_step=epoch * len(train_data_loader) + batch_id,\n", " global_step=epoch * len(train_data_loader) + batch_id,\n",
" )\n", " )\n",
" loss.backward()\n", " loss.backward()\n",
"\n",
" if hyperparameters[\"clip_gradients\"]:\n",
" clip_grad_norm_(model.parameters(), max_norm=1.0)\n",
" optimizer.step()\n", " optimizer.step()\n",
"\n", "\n",
" logging.info(f\"Epoch {epoch} train loss: {epoch_loss}\")\n", " logging.info(f\"Epoch {epoch} train loss: {epoch_loss}\")\n",
@ -226,18 +195,12 @@
" model.eval()\n", " model.eval()\n",
" loader = iter(test_data_loader)\n", " loader = iter(test_data_loader)\n",
" edited_histogram, original_histogram = next(loader)\n", " edited_histogram, original_histogram = next(loader)\n",
" edited_histogram = edited_histogram.to(device)\n", " predicted_original = model(edited_histogram.to(device))\n",
" original_histogram = original_histogram.to(device)\n",
" predicted_original = model(edited_histogram)\n",
" sum = torch.sum(predicted_original, dim=(2, 3, 4), keepdim=True)\n",
" predicted_original = predicted_original / sum\n",
" writer.add_figure(\n", " writer.add_figure(\n",
" \"histogram\",\n", " \"histogram\",\n",
" plot_histograms_in_2d(\n", " plot_histograms_in_2d(\n",
" {\n", " {\n",
" \"original\": original_histogram.cpu()[0]\n", " \"original\": original_histogram[0].numpy().squeeze(),\n",
" .numpy()\n",
" .squeeze(),\n",
" \"edited\": edited_histogram.cpu()[0].numpy().squeeze(),\n", " \"edited\": edited_histogram.cpu()[0].numpy().squeeze(),\n",
" \"predicted\": predicted_original.cpu()[0]\n", " \"predicted\": predicted_original.cpu()[0]\n",
" .numpy()\n", " .numpy()\n",
@ -251,29 +214,11 @@
" for batch_id, (edited_histogram, original_histogram) in enumerate(\n", " for batch_id, (edited_histogram, original_histogram) in enumerate(\n",
" test_data_loader\n", " test_data_loader\n",
" ):\n", " ):\n",
" edited_histogram = edited_histogram.to(device)\n", " predicted_original = model(edited_histogram.to(device))\n",
" original_histogram = original_histogram.to(device)\n", " epoch_test_loss += loss_function(\n",
"\n", " torch.log(torch.clamp(predicted_original, 1e-10, 1)),\n",
" predicted_original = model(edited_histogram)\n", " original_histogram.to(device),\n",
" sum = torch.sum(predicted_original, dim=(2, 3, 4), keepdim=True)\n", " ).item()\n",
" predicted_original = predicted_original / sum\n",
"\n",
" if hyperparameters[\"loss\"] == \"kl\":\n",
" predicted_original = torch.clamp(\n",
" predicted_original, 0.0000000000000000000001, 1\n",
" )\n",
"\n",
" loss = {\n",
" \"kl\": lambda: loss_function(\n",
" torch.log(predicted_original),\n",
" original_histogram,\n",
" ),\n",
" \"progressive\": lambda: loss_function(\n",
" predicted_original, original_histogram\n",
" ),\n",
" }[hyperparameters[\"loss\"]]()\n",
"\n",
" epoch_test_loss += loss.item()\n",
" writer.add_hparams(\n", " writer.add_hparams(\n",
" serialise_hparams(hyperparameters),\n", " serialise_hparams(hyperparameters),\n",
" {\n", " {\n",
@ -287,40 +232,323 @@
"\n", "\n",
" model.train()\n", " model.train()\n",
" scheduler.step()\n", " scheduler.step()\n",
" except Exception:\n",
" raise\n",
" finally:\n", " finally:\n",
" run_name = get_next_run_name(MODELS_PATH)\n", " model_path = MODELS_PATH / get_next_run_name(MODELS_PATH)\n",
" model_path = (MODELS_PATH / run_name).with_suffix(\".pth\")\n", " save_model(model, hyperparameters, model_path)\n",
" params_path = (MODELS_PATH / run_name).with_suffix(\".json\")\n",
"\n",
" logging.info(f\"Saving model to {model_path}\")\n",
" with open(model_path, \"wb\") as f:\n",
" torch.save(model.state_dict(), f)\n",
" with open(params_path, \"w\") as f:\n",
" json.dump(hyperparameters, f, indent=2)\n",
" del model\n", " del model\n",
" torch.cuda.empty_cache()\n",
" return model_path" " return model_path"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 5, "execution_count": 7,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"2024-06-22 16:57:28,986 - INFO - Loaded 22479 original images\n",
"2024-06-22 16:57:28,991 - INFO - Loaded 2498 original images\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "63e068d857484e59afb823ca4b5c3a58",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Epoch 0: 0%| | 0/2108 [00:00<?, ?batch/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"2024-06-22 17:00:34,218 - INFO - Epoch 0 train loss: 11718.475350141525\n",
"/home/andras/miniconda3/envs/bipolaroid/lib/python3.12/site-packages/matplotlib/collections.py:996: RuntimeWarning: invalid value encountered in sqrt\n",
" scale = np.sqrt(self._sizes) * dpi / 72.0 * self._factor\n",
"2024-06-22 17:00:43,669 - INFO - Epoch 0 test loss: 575.4344878196716\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "594bc7c470a149b4a1c89a5728d67ab1",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Epoch 1: 0%| | 0/2108 [00:00<?, ?batch/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"2024-06-22 17:03:46,881 - INFO - Epoch 1 train loss: 9741.187401413918\n",
"2024-06-22 17:03:56,471 - INFO - Epoch 1 test loss: 536.2769713401794\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "830ea2fbef8d4e1c8e45650f82560c74",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Epoch 2: 0%| | 0/2108 [00:00<?, ?batch/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"2024-06-22 17:06:59,896 - INFO - Epoch 2 train loss: 9120.070751070976\n",
"2024-06-22 17:07:09,641 - INFO - Epoch 2 test loss: 553.2901458740234\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "077a671090584e97a88adbfc35007919",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Epoch 3: 0%| | 0/2108 [00:00<?, ?batch/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"2024-06-22 17:10:12,942 - INFO - Epoch 3 train loss: 5763.117876529694\n",
"2024-06-22 17:10:22,622 - INFO - Epoch 3 test loss: 507.6950304508209\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "786b050c3f85486096d84816aab6affd",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Epoch 4: 0%| | 0/2108 [00:00<?, ?batch/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"2024-06-22 17:13:25,908 - INFO - Epoch 4 train loss: 6363.094870328903\n",
"2024-06-22 17:13:36,532 - INFO - Epoch 4 test loss: 532.4468264579773\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "070b1d8d19ec4958b9d1175cef666802",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Epoch 5: 0%| | 0/2108 [00:00<?, ?batch/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"2024-06-22 17:16:40,056 - INFO - Epoch 5 train loss: 4596.043945550919\n",
"2024-06-22 17:16:49,784 - INFO - Epoch 5 test loss: 438.763400554657\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "5531f44fbc84422e9739dd94121b100f",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Epoch 6: 0%| | 0/2108 [00:00<?, ?batch/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"2024-06-22 17:19:53,205 - INFO - Epoch 6 train loss: 5266.503381967545\n",
"2024-06-22 17:20:02,990 - INFO - Epoch 6 test loss: 573.5293898582458\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "a73cdeb982ab418185fde63e80fe07db",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Epoch 7: 0%| | 0/2108 [00:00<?, ?batch/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"2024-06-22 17:23:06,445 - INFO - Epoch 7 train loss: 5163.991681098938\n",
"2024-06-22 17:23:16,136 - INFO - Epoch 7 test loss: 672.4951323270798\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "8f04a8e6f5324b688ce8a6c9f4453f78",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Epoch 8: 0%| | 0/2108 [00:00<?, ?batch/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"2024-06-22 17:26:19,453 - INFO - Epoch 8 train loss: 12930.857147455215\n",
"2024-06-22 17:26:29,204 - INFO - Epoch 8 test loss: 636.4001806974411\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "6dee2d0ebb564ff89604ad8a81787998",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Epoch 9: 0%| | 0/2108 [00:00<?, ?batch/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"2024-06-22 17:29:32,560 - INFO - Epoch 9 train loss: 13841.072596549988\n",
"2024-06-22 17:29:42,246 - INFO - Epoch 9 test loss: 2833.8614711761475\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "0b4729e7da054a3c8b2e0609459a18a7",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Epoch 10: 0%| | 0/2108 [00:00<?, ?batch/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"2024-06-22 17:32:45,585 - INFO - Epoch 10 train loss: 15531.006411075592\n",
"2024-06-22 17:32:55,355 - INFO - Epoch 10 test loss: 469.56569051742554\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "9fda0f52c620475b927bbff090b048d5",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Epoch 11: 0%| | 0/2108 [00:00<?, ?batch/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"2024-06-22 17:35:58,670 - INFO - Epoch 11 train loss: 17766.0949113369\n",
"2024-06-22 17:36:08,527 - INFO - Epoch 11 test loss: 3254.2825841903687\n",
"2024-06-22 17:36:08,529 - INFO - Saving model to /home/andras/projects/bipolaroid/models/run_66.pth\n",
"2024-06-22 17:36:08,529 - INFO - Parameter count: 429457\n"
]
},
{
"data": {
"text/plain": [
"PosixPath('/home/andras/projects/bipolaroid/models/run_66')"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [ "source": [
"# train(\n", "# train(\n",
"# {\n", "# {\n",
"# \"batch_size\": 64,\n", "# \"batch_size\": 128,\n",
"# \"edit_count\": 8,\n", "# \"edit_count\": 12,\n",
"# \"bin_count\": 16,\n", "# \"bin_count\": 16,\n",
"# \"clip_gradients\": False,\n", "# \"learning_rate\": 1e-3,\n",
"# \"learning_rate\": 0.0005220900529274365,\n", "# \"scheduler_gamma\": 0.8,\n",
"# \"scheduler_gamma\": 0.5479991284291021,\n", "# \"elu_alpha\": 1,\n",
"# \"num_epochs\": 24,\n", "# \"dropout_prob\": 0.05,\n",
"# \"model_type\": \"Residual\",\n", "# \"features\": [8, 16, 32],\n",
"# \"loss\": \"kl\",\n", "# \"kernel_sizes\": [3, 3, 3],\n",
"# \"num_epochs\": 12,\n",
"# \"model_type\": \"Residual3\",\n",
"# \"clip_gradients\": True,\n",
"# \"use_instance_norm\": True,\n",
"# \"use_elu\": False,\n",
"# \"leaky_relu_alpha\": 0.01,\n",
"# }\n", "# }\n",
"# )" "# )"
] ]
@ -329,45 +557,7 @@
"cell_type": "code", "cell_type": "code",
"execution_count": 6, "execution_count": 6,
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [],
{
"name": "stderr",
"output_type": "stream",
"text": [
"2024-06-16 20:21:51,962 - INFO - Starting run_0 with hparams {\n",
" \"batch_size\": 64,\n",
" \"bin_count\": 32,\n",
" \"clip_gradients\": false,\n",
" \"edit_count\": 8,\n",
" \"learning_rate\": 0.0013249692770052317,\n",
" \"loss\": \"kl\",\n",
" \"model_type\": \"Residual\",\n",
" \"num_epochs\": 16,\n",
" \"scheduler_gamma\": 1.3114281184948258\n",
"}\n",
"2024-06-16 20:21:52,012 - INFO - Loaded 22479 original images\n",
"2024-06-16 20:21:52,016 - INFO - Loaded 2498 original images\n",
"2024-06-16 20:35:43,995 - INFO - Epoch 0 train loss: 6540.840226650238\n",
"2024-06-16 20:36:15,017 - INFO - Epoch 0 test loss: 1531.6546006202698\n",
"2024-06-16 20:49:58,543 - INFO - Epoch 1 train loss: 5763.938045859337\n",
"2024-06-16 20:50:29,893 - INFO - Epoch 1 test loss: 1608.853798866272\n",
"2024-06-16 21:04:13,577 - INFO - Epoch 2 train loss: 5448.952376246452\n",
"2024-06-16 21:04:45,607 - INFO - Epoch 2 test loss: 1465.128571987152\n",
"2024-06-16 21:18:31,962 - INFO - Epoch 3 train loss: 5633.2793600559235\n",
"2024-06-16 21:19:09,149 - INFO - Epoch 3 test loss: 1330.329261302948\n",
"2024-06-16 21:32:58,465 - INFO - Epoch 4 train loss: 5338.784257531166\n",
"2024-06-16 21:33:37,006 - INFO - Epoch 4 test loss: 2083.3998107910156\n",
"2024-06-16 21:47:25,527 - INFO - Epoch 5 train loss: 5321.843332529068\n",
"2024-06-16 21:48:04,110 - INFO - Epoch 5 test loss: 1314.629390001297\n",
"2024-06-16 22:01:51,264 - INFO - Epoch 6 train loss: 5337.748890757561\n",
"2024-06-16 22:02:29,786 - INFO - Epoch 6 test loss: 1290.2974362373352\n",
"2024-06-16 22:16:17,529 - INFO - Epoch 7 train loss: 5167.580719232559\n",
"2024-06-16 22:16:56,284 - INFO - Epoch 7 test loss: 1889.667366027832\n",
"2024-06-16 22:26:15,914 - INFO - Saving model to /home/andras/projects/bipolaroid/models/run_67.pth\n",
"2024-06-16 22:26:16,086 - INFO - Interrupted, stopping\n"
]
}
],
"source": [ "source": [
"from random import choice\n", "from random import choice\n",
"from itertools import count\n", "from itertools import count\n",
@ -380,9 +570,7 @@
" for k, v in choice(hyperparameters).items()\n", " for k, v in choice(hyperparameters).items()\n",
" }\n", " }\n",
" key = json.dumps(current_hyperparameters, indent=2, sort_keys=True)\n", " key = json.dumps(current_hyperparameters, indent=2, sort_keys=True)\n",
" logging.info(\n", " logging.info(f\"Starting {get_next_run_name(RUNS_PATH)} with hparams {key}\")\n",
" f\"Starting {get_next_run_name(RUNS_PATH)} with hparams {key}\"\n",
" )\n",
" try:\n", " try:\n",
" train(current_hyperparameters, max_duration=timedelta(hours=8), use_tqdm=False)\n", " train(current_hyperparameters, max_duration=timedelta(hours=8), use_tqdm=False)\n",
" except KeyboardInterrupt as e:\n", " except KeyboardInterrupt as e:\n",
@ -413,7 +601,7 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.1.-1" "version": "3.12.2"
} }
}, },
"nbformat": 4, "nbformat": 4,