570 lines
25 KiB
Text
570 lines
25 KiB
Text
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"metadata": {
|
|
"metadata": {}
|
|
},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"2024-06-03 07:46:08,999 - INFO - PyTorch version: 2.2.2\n"
|
|
]
|
|
},
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"'Using device cuda:0'"
|
|
]
|
|
},
|
|
"execution_count": 2,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"import torch\n",
|
|
"import logging\n",
|
|
"import os\n",
|
|
"from datetime import datetime\n",
|
|
"\n",
|
|
"logging.basicConfig(\n",
|
|
" level=logging.INFO,\n",
|
|
" format=\"%(asctime)s - %(levelname)s - %(message)s\",\n",
|
|
" handlers=[\n",
|
|
" logging.StreamHandler(),\n",
|
|
" logging.FileHandler(f\"train-{datetime.now().date()}.log\"),\n",
|
|
" ],\n",
|
|
")\n",
|
|
"\n",
|
|
"os.environ[\"PYTORCH_CUDA_ALLOC_CONF\"] = \"expandable_segments:True\"\n",
|
|
"device = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
|
|
"f\"Using device {device}\""
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from scipy.stats import loguniform, uniform\n",
|
|
"from editor.models import MODELS\n",
|
|
"\n",
|
|
"common_hyperparameters = {\n",
|
|
" \"batch_size\": [16, 32, 64],\n",
|
|
" \"edit_count\": [8, 16],\n",
|
|
" \"bin_count\": [16, 32],\n",
|
|
" \"clip_gradients\": [True, False],\n",
|
|
" \"learning_rate\": loguniform(0.0001, 0.005),\n",
|
|
" \"scheduler_gamma\": uniform(0.1, 0.9),\n",
|
|
" \"num_epochs\": [5],\n",
|
|
" \"model_type\": list(MODELS.keys()),\n",
|
|
"}\n",
|
|
"hyperparameters = [\n",
|
|
" # {\n",
|
|
" # **common_hyperparameters,\n",
|
|
" # \"loss\": [\"progressive\"],\n",
|
|
" # \"loss_sizes\": [[4, 8, 16, 32], [8, 16, 32], [16, 32], [8, 32]],\n",
|
|
" # \"loss_damping\": uniform(0.2, 5),\n",
|
|
" # },\n",
|
|
" {\n",
|
|
" **common_hyperparameters,\n",
|
|
" \"loss\": [\"kl\"],\n",
|
|
" },\n",
|
|
"]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Testing model SimpleCNN\n",
|
|
"Test passed! Output shape matches input shape.\n",
|
|
"Testing model Residual\n",
|
|
"Test passed! Output shape matches input shape.\n",
|
|
"Testing model NormalisedCNN\n",
|
|
"Test passed! Output shape matches input shape.\n",
|
|
"Testing model SmartRes\n",
|
|
"Test passed! Output shape matches input shape.\n",
|
|
"Testing model attention2\n",
|
|
"Test passed! Output shape matches input shape.\n",
|
|
"Testing model advanced_attention\n",
|
|
"Test passed! Output shape matches input shape.\n",
|
|
"Testing model Res2\n",
|
|
"Test passed! Output shape matches input shape.\n",
|
|
"Testing model attention1\n",
|
|
"Test passed! Output shape matches input shape.\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"from typing import Any, Dict\n",
|
|
"from torch.utils.tensorboard import SummaryWriter\n",
|
|
"from pathlib import Path\n",
|
|
"from torch.optim import Adam\n",
|
|
"from tqdm.notebook import tqdm\n",
|
|
"from torch.nn.utils import clip_grad_norm_\n",
|
|
"from editor.training import ProgressivePoolingLoss\n",
|
|
"from editor.utils import get_next_run_name\n",
|
|
"from editor.visualisation import plot_histograms_in_2d\n",
|
|
"from editor.training import create_data_loaders\n",
|
|
"from editor.models import create_model, test_models\n",
|
|
"from config import DATA, MODELS_PATH\n",
|
|
"from datetime import timedelta, datetime\n",
|
|
"\n",
|
|
"test_models()\n",
|
|
"\n",
|
|
"\n",
|
|
"def train(\n",
|
|
" hyperparameters: Dict[str, Any], max_duration: timedelta, use_tqdm: bool\n",
|
|
") -> Path:\n",
|
|
" start_time = datetime.now()\n",
|
|
" model_path = (MODELS_PATH / get_next_run_name(Path(\"runs\"))).with_suffix(\".pth\")\n",
|
|
"\n",
|
|
" log_dir = Path(\"runs\") / get_next_run_name(Path(\"runs\"))\n",
|
|
" with SummaryWriter(log_dir) as writer:\n",
|
|
" train_data_loader, test_data_loader = create_data_loaders(\n",
|
|
" data=DATA,\n",
|
|
" edit_count=hyperparameters[\"edit_count\"],\n",
|
|
" bin_count=hyperparameters[\"bin_count\"],\n",
|
|
" training_batch_size=hyperparameters[\"batch_size\"],\n",
|
|
" )\n",
|
|
"\n",
|
|
" model = (\n",
|
|
" create_model(\n",
|
|
" type=hyperparameters[\"model_type\"],\n",
|
|
" bin_count=hyperparameters[\"bin_count\"],\n",
|
|
" )\n",
|
|
" .train()\n",
|
|
" .to(device)\n",
|
|
" )\n",
|
|
" writer.add_graph(model, next(iter(train_data_loader))[0].to(device))\n",
|
|
"\n",
|
|
" optimizer = Adam(model.parameters(), lr=hyperparameters[\"learning_rate\"])\n",
|
|
" scheduler = torch.optim.lr_scheduler.StepLR(\n",
|
|
" optimizer, step_size=1, gamma=hyperparameters[\"scheduler_gamma\"]\n",
|
|
" )\n",
|
|
"\n",
|
|
" loss_function = {\n",
|
|
" \"progressive\": lambda: ProgressivePoolingLoss(\n",
|
|
" target_sizes=hyperparameters[\"loss_sizes\"],\n",
|
|
" damping=hyperparameters[\"loss_damping\"],\n",
|
|
" ),\n",
|
|
" \"kl\": lambda: torch.nn.KLDivLoss(reduction=\"batchmean\"),\n",
|
|
" }[hyperparameters[\"loss\"]]().to(device)\n",
|
|
"\n",
|
|
" try:\n",
|
|
" for epoch in range(hyperparameters[\"num_epochs\"]):\n",
|
|
" epoch_loss = 0\n",
|
|
" writer.add_scalar(\n",
|
|
" \"Actual learning rate\", scheduler.get_last_lr()[0], epoch\n",
|
|
" )\n",
|
|
" for batch_id, (edited_histogram, original_histogram) in enumerate(\n",
|
|
" tqdm(train_data_loader, desc=f\"Epoch {epoch}\", unit=\"batch\")\n",
|
|
" if use_tqdm\n",
|
|
" else train_data_loader\n",
|
|
" ):\n",
|
|
" current_time = datetime.now()\n",
|
|
" if current_time - start_time > max_duration:\n",
|
|
" raise TimeoutError(f\"Time limit {max_duration} exceeded\")\n",
|
|
" edited_histogram = edited_histogram.to(device)\n",
|
|
" original_histogram = original_histogram.to(device)\n",
|
|
"\n",
|
|
" optimizer.zero_grad()\n",
|
|
" predicted_original = model(edited_histogram)\n",
|
|
" sum = torch.sum(predicted_original, dim=(2, 3, 4), keepdim=True)\n",
|
|
" predicted_original = predicted_original / sum\n",
|
|
"\n",
|
|
" if hyperparameters[\"loss\"] == \"kl\":\n",
|
|
" predicted_original = torch.clamp(\n",
|
|
" predicted_original, 0.0000000000000000000001, 1\n",
|
|
" )\n",
|
|
"\n",
|
|
" loss = {\n",
|
|
" \"kl\": lambda: loss_function(\n",
|
|
" torch.log(predicted_original),\n",
|
|
" original_histogram,\n",
|
|
" ),\n",
|
|
" \"progressive\": lambda: loss_function(\n",
|
|
" predicted_original, original_histogram\n",
|
|
" ),\n",
|
|
" }[hyperparameters[\"loss\"]]()\n",
|
|
"\n",
|
|
" epoch_loss += loss.item()\n",
|
|
" writer.add_scalar(\n",
|
|
" \"Loss/train/batch\",\n",
|
|
" loss,\n",
|
|
" epoch * len(train_data_loader) + batch_id,\n",
|
|
" )\n",
|
|
" loss.backward()\n",
|
|
"\n",
|
|
" if hyperparameters[\"clip_gradients\"]:\n",
|
|
" clip_grad_norm_(model.parameters(), max_norm=1.0)\n",
|
|
" optimizer.step()\n",
|
|
"\n",
|
|
" writer.add_hparams(\n",
|
|
" {\n",
|
|
" k: str(v) if isinstance(v, list) else v\n",
|
|
" for k, v in hyperparameters.items()\n",
|
|
" },\n",
|
|
" {\n",
|
|
" \"Loss/train/epoch\": epoch_loss,\n",
|
|
" },\n",
|
|
" global_step=epoch,\n",
|
|
" run_name=log_dir.absolute(),\n",
|
|
" )\n",
|
|
" with torch.no_grad():\n",
|
|
" model.eval()\n",
|
|
" loader = iter(test_data_loader)\n",
|
|
" edited_histogram, original_histogram = next(loader)\n",
|
|
" edited_histogram = edited_histogram.to(device)\n",
|
|
" original_histogram = original_histogram.to(device)\n",
|
|
" predicted_original = model(edited_histogram)\n",
|
|
" sum = torch.sum(predicted_original, dim=(2, 3, 4), keepdim=True)\n",
|
|
" predicted_original = predicted_original / sum\n",
|
|
" writer.add_figure(\n",
|
|
" \"histogram\",\n",
|
|
" plot_histograms_in_2d(\n",
|
|
" {\n",
|
|
" \"original\": original_histogram.cpu()[0]\n",
|
|
" .numpy()\n",
|
|
" .squeeze(),\n",
|
|
" \"edited\": edited_histogram.cpu()[0].numpy().squeeze(),\n",
|
|
" \"predicted\": predicted_original.cpu()[0]\n",
|
|
" .numpy()\n",
|
|
" .squeeze(),\n",
|
|
" }\n",
|
|
" ),\n",
|
|
" epoch,\n",
|
|
" )\n",
|
|
" model.train()\n",
|
|
" scheduler.step()\n",
|
|
" except Exception as e:\n",
|
|
" raise\n",
|
|
" finally:\n",
|
|
" torch.save(model.state_dict(), model_path)\n",
|
|
" del model\n",
|
|
" torch.cuda.empty_cache()\n",
|
|
" return model_path"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# train(\n",
|
|
"# {\n",
|
|
"# \"batch_size\": 64,\n",
|
|
"# \"edit_count\": 25,\n",
|
|
"# \"bin_count\": 32,\n",
|
|
"# \"clip_gradients\": True,\n",
|
|
"# \"learning_rate\": 0.005,\n",
|
|
"# \"scheduler_gamma\": 0.7,\n",
|
|
"# \"num_epochs\": 20,\n",
|
|
"# \"model_type\": \"NormalisedCNN\",\n",
|
|
"# \"loss\": \"progressive\",\n",
|
|
"# \"loss_sizes\": [16, 32],\n",
|
|
"# \"loss_damping\": 2,\n",
|
|
"# }\n",
|
|
"# )"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"2024-06-02 21:42:49,762 - INFO - Starting run_51 with hparams {\n",
|
|
" \"batch_size\": 16,\n",
|
|
" \"bin_count\": 64,\n",
|
|
" \"clip_gradients\": true,\n",
|
|
" \"edit_count\": 16,\n",
|
|
" \"learning_rate\": 0.0019018860481580008,\n",
|
|
" \"loss\": \"kl\",\n",
|
|
" \"model_type\": \"Residual\",\n",
|
|
" \"num_epochs\": 10,\n",
|
|
" \"scheduler_gamma\": 0.5124233085818609\n",
|
|
"}\n",
|
|
"2024-06-02 21:42:49,787 - INFO - Loaded 359668 training images and 39964 test images\n",
|
|
"2024-06-02 23:43:03,467 - WARNING - Timeout, aborting experiment\n",
|
|
"2024-06-02 23:43:03,698 - INFO - Starting run_52 with hparams {\n",
|
|
" \"batch_size\": 16,\n",
|
|
" \"bin_count\": 16,\n",
|
|
" \"clip_gradients\": false,\n",
|
|
" \"edit_count\": 8,\n",
|
|
" \"learning_rate\": 2.9976475506468536e-05,\n",
|
|
" \"loss\": \"kl\",\n",
|
|
" \"model_type\": \"SmartRes\",\n",
|
|
" \"num_epochs\": 10,\n",
|
|
" \"scheduler_gamma\": 0.8138813825657673\n",
|
|
"}\n",
|
|
"2024-06-02 23:43:03,991 - INFO - Loaded 179834 training images and 19982 test images\n",
|
|
"/home/andras/miniconda3/envs/bipolaroid/lib/python3.12/site-packages/matplotlib/collections.py:996: RuntimeWarning: invalid value encountered in sqrt\n",
|
|
" scale = np.sqrt(self._sizes) * dpi / 72.0 * self._factor\n",
|
|
"2024-06-02 23:52:17,393 - INFO - Starting run_53 with hparams {\n",
|
|
" \"batch_size\": 8,\n",
|
|
" \"bin_count\": 32,\n",
|
|
" \"clip_gradients\": false,\n",
|
|
" \"edit_count\": 8,\n",
|
|
" \"learning_rate\": 0.0002765101396434423,\n",
|
|
" \"loss\": \"kl\",\n",
|
|
" \"model_type\": \"SmartRes\",\n",
|
|
" \"num_epochs\": 10,\n",
|
|
" \"scheduler_gamma\": 0.8393595799921102\n",
|
|
"}\n",
|
|
"2024-06-02 23:52:17,413 - INFO - Loaded 179834 training images and 19982 test images\n",
|
|
"2024-06-03 00:48:49,485 - INFO - Starting run_54 with hparams {\n",
|
|
" \"batch_size\": 16,\n",
|
|
" \"bin_count\": 16,\n",
|
|
" \"clip_gradients\": false,\n",
|
|
" \"edit_count\": 16,\n",
|
|
" \"learning_rate\": 0.00040493280785202865,\n",
|
|
" \"loss\": \"kl\",\n",
|
|
" \"model_type\": \"SmartRes\",\n",
|
|
" \"num_epochs\": 10,\n",
|
|
" \"scheduler_gamma\": 0.6647838946959123\n",
|
|
"}\n",
|
|
"2024-06-03 00:48:49,509 - INFO - Loaded 359668 training images and 39964 test images\n",
|
|
"2024-06-03 01:10:40,678 - INFO - Starting run_55 with hparams {\n",
|
|
" \"batch_size\": 32,\n",
|
|
" \"bin_count\": 16,\n",
|
|
" \"clip_gradients\": true,\n",
|
|
" \"edit_count\": 16,\n",
|
|
" \"learning_rate\": 0.000989324245186775,\n",
|
|
" \"loss\": \"kl\",\n",
|
|
" \"model_type\": \"SmartRes\",\n",
|
|
" \"num_epochs\": 10,\n",
|
|
" \"scheduler_gamma\": 0.6779989111474544\n",
|
|
"}\n",
|
|
"2024-06-03 01:10:40,704 - INFO - Loaded 359668 training images and 39964 test images\n",
|
|
"2024-06-03 01:26:06,028 - INFO - Starting run_56 with hparams {\n",
|
|
" \"batch_size\": 8,\n",
|
|
" \"bin_count\": 16,\n",
|
|
" \"clip_gradients\": false,\n",
|
|
" \"edit_count\": 16,\n",
|
|
" \"learning_rate\": 1.0695951486573912e-05,\n",
|
|
" \"loss\": \"kl\",\n",
|
|
" \"model_type\": \"Residual\",\n",
|
|
" \"num_epochs\": 10,\n",
|
|
" \"scheduler_gamma\": 0.3619561054933521\n",
|
|
"}\n",
|
|
"2024-06-03 01:26:06,052 - INFO - Loaded 359668 training images and 39964 test images\n",
|
|
"2024-06-03 02:03:39,558 - INFO - Starting run_57 with hparams {\n",
|
|
" \"batch_size\": 32,\n",
|
|
" \"bin_count\": 64,\n",
|
|
" \"clip_gradients\": false,\n",
|
|
" \"edit_count\": 16,\n",
|
|
" \"learning_rate\": 0.00024721579172106914,\n",
|
|
" \"loss\": \"kl\",\n",
|
|
" \"model_type\": \"attention1\",\n",
|
|
" \"num_epochs\": 10,\n",
|
|
" \"scheduler_gamma\": 0.7999479970967494\n",
|
|
"}\n",
|
|
"2024-06-03 02:03:39,585 - INFO - Loaded 359668 training images and 39964 test images\n",
|
|
"2024-06-03 02:05:40,747 - ERROR - Error with hparams {'batch_size': 32, 'edit_count': 16, 'bin_count': 64, 'clip_gradients': False, 'learning_rate': 0.00024721579172106914, 'scheduler_gamma': 0.7999479970967494, 'num_epochs': 10, 'model_type': 'attention1', 'loss': 'kl'}:\n",
|
|
"\tCUDA out of memory. Tried to allocate 6.00 GiB. GPU 0 has a total capacity of 15.99 GiB of which 0 bytes is free. Including non-PyTorch memory, this process has 17179869184.00 GiB memory in use. Of the allocated memory 39.04 GiB is allocated by PyTorch, and 2.02 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n",
|
|
"Stack (most recent call last):\n",
|
|
" File \"<frozen runpy>\", line 198, in _run_module_as_main\n",
|
|
" File \"<frozen runpy>\", line 88, in _run_code\n",
|
|
" File \"/home/andras/miniconda3/envs/bipolaroid/lib/python3.12/site-packages/ipykernel_launcher.py\", line 18, in <module>\n",
|
|
" app.launch_new_instance()\n",
|
|
" File \"/home/andras/miniconda3/envs/bipolaroid/lib/python3.12/site-packages/traitlets/config/application.py\", line 1075, in launch_instance\n",
|
|
" app.start()\n",
|
|
" File \"/home/andras/miniconda3/envs/bipolaroid/lib/python3.12/site-packages/ipykernel/kernelapp.py\", line 739, in start\n",
|
|
" self.io_loop.start()\n",
|
|
" File \"/home/andras/miniconda3/envs/bipolaroid/lib/python3.12/site-packages/tornado/platform/asyncio.py\", line 195, in start\n",
|
|
" self.asyncio_loop.run_forever()\n",
|
|
" File \"/home/andras/miniconda3/envs/bipolaroid/lib/python3.12/asyncio/base_events.py\", line 639, in run_forever\n",
|
|
" self._run_once()\n",
|
|
" File \"/home/andras/miniconda3/envs/bipolaroid/lib/python3.12/asyncio/base_events.py\", line 1985, in _run_once\n",
|
|
" handle._run()\n",
|
|
" File \"/home/andras/miniconda3/envs/bipolaroid/lib/python3.12/asyncio/events.py\", line 88, in _run\n",
|
|
" self._context.run(self._callback, *self._args)\n",
|
|
" File \"/home/andras/miniconda3/envs/bipolaroid/lib/python3.12/site-packages/ipykernel/kernelbase.py\", line 545, in dispatch_queue\n",
|
|
" await self.process_one()\n",
|
|
" File \"/home/andras/miniconda3/envs/bipolaroid/lib/python3.12/site-packages/ipykernel/kernelbase.py\", line 534, in process_one\n",
|
|
" await dispatch(*args)\n",
|
|
" File \"/home/andras/miniconda3/envs/bipolaroid/lib/python3.12/site-packages/ipykernel/kernelbase.py\", line 437, in dispatch_shell\n",
|
|
" await result\n",
|
|
" File \"/home/andras/miniconda3/envs/bipolaroid/lib/python3.12/site-packages/ipykernel/ipkernel.py\", line 359, in execute_request\n",
|
|
" await super().execute_request(stream, ident, parent)\n",
|
|
" File \"/home/andras/miniconda3/envs/bipolaroid/lib/python3.12/site-packages/ipykernel/kernelbase.py\", line 778, in execute_request\n",
|
|
" reply_content = await reply_content\n",
|
|
" File \"/home/andras/miniconda3/envs/bipolaroid/lib/python3.12/site-packages/ipykernel/ipkernel.py\", line 446, in do_execute\n",
|
|
" res = shell.run_cell(\n",
|
|
" File \"/home/andras/miniconda3/envs/bipolaroid/lib/python3.12/site-packages/ipykernel/zmqshell.py\", line 549, in run_cell\n",
|
|
" return super().run_cell(*args, **kwargs)\n",
|
|
" File \"/home/andras/miniconda3/envs/bipolaroid/lib/python3.12/site-packages/IPython/core/interactiveshell.py\", line 3075, in run_cell\n",
|
|
" result = self._run_cell(\n",
|
|
" File \"/home/andras/miniconda3/envs/bipolaroid/lib/python3.12/site-packages/IPython/core/interactiveshell.py\", line 3130, in _run_cell\n",
|
|
" result = runner(coro)\n",
|
|
" File \"/home/andras/miniconda3/envs/bipolaroid/lib/python3.12/site-packages/IPython/core/async_helpers.py\", line 129, in _pseudo_sync_runner\n",
|
|
" coro.send(None)\n",
|
|
" File \"/home/andras/miniconda3/envs/bipolaroid/lib/python3.12/site-packages/IPython/core/interactiveshell.py\", line 3334, in run_cell_async\n",
|
|
" has_raised = await self.run_ast_nodes(code_ast.body, cell_name,\n",
|
|
" File \"/home/andras/miniconda3/envs/bipolaroid/lib/python3.12/site-packages/IPython/core/interactiveshell.py\", line 3517, in run_ast_nodes\n",
|
|
" if await self.run_code(code, result, async_=asy):\n",
|
|
" File \"/home/andras/miniconda3/envs/bipolaroid/lib/python3.12/site-packages/IPython/core/interactiveshell.py\", line 3577, in run_code\n",
|
|
" exec(code_obj, self.user_global_ns, self.user_ns)\n",
|
|
" File \"/tmp/ipykernel_141525/1542138470.py\", line 28, in <module>\n",
|
|
" logging.error(\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"CUDA out of memory. Tried to allocate 6.00 GiB. GPU 0 has a total capacity of 15.99 GiB of which 0 bytes is free. Including non-PyTorch memory, this process has 17179869184.00 GiB memory in use. Of the allocated memory 39.04 GiB is allocated by PyTorch, and 2.02 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n",
|
|
"Error occurs, No graph saved\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"2024-06-03 02:05:41,071 - INFO - Starting run_58 with hparams {\n",
|
|
" \"batch_size\": 64,\n",
|
|
" \"bin_count\": 16,\n",
|
|
" \"clip_gradients\": false,\n",
|
|
" \"edit_count\": 16,\n",
|
|
" \"learning_rate\": 5.8262398455352215e-05,\n",
|
|
" \"loss\": \"kl\",\n",
|
|
" \"model_type\": \"attention2\",\n",
|
|
" \"num_epochs\": 10,\n",
|
|
" \"scheduler_gamma\": 0.17181073763193916\n",
|
|
"}\n",
|
|
"2024-06-03 02:05:41,262 - INFO - Loaded 359668 training images and 39964 test images\n",
|
|
"2024-06-03 03:49:02,268 - INFO - Starting run_59 with hparams {\n",
|
|
" \"batch_size\": 16,\n",
|
|
" \"bin_count\": 16,\n",
|
|
" \"clip_gradients\": false,\n",
|
|
" \"edit_count\": 32,\n",
|
|
" \"learning_rate\": 0.00017213076448986518,\n",
|
|
" \"loss\": \"kl\",\n",
|
|
" \"model_type\": \"NormalisedCNN\",\n",
|
|
" \"num_epochs\": 10,\n",
|
|
" \"scheduler_gamma\": 0.1302383221350669\n",
|
|
"}\n",
|
|
"2024-06-03 03:49:02,397 - INFO - Loaded 719337 training images and 79927 test images\n",
|
|
"2024-06-03 04:28:45,612 - INFO - Starting run_60 with hparams {\n",
|
|
" \"batch_size\": 16,\n",
|
|
" \"bin_count\": 16,\n",
|
|
" \"clip_gradients\": false,\n",
|
|
" \"edit_count\": 32,\n",
|
|
" \"learning_rate\": 0.00010975854085067054,\n",
|
|
" \"loss\": \"kl\",\n",
|
|
" \"model_type\": \"NormalisedCNN\",\n",
|
|
" \"num_epochs\": 10,\n",
|
|
" \"scheduler_gamma\": 0.5457536006732233\n",
|
|
"}\n",
|
|
"2024-06-03 04:28:45,645 - INFO - Loaded 719337 training images and 79927 test images\n",
|
|
"2024-06-03 05:07:36,501 - INFO - Starting run_61 with hparams {\n",
|
|
" \"batch_size\": 16,\n",
|
|
" \"bin_count\": 32,\n",
|
|
" \"clip_gradients\": false,\n",
|
|
" \"edit_count\": 16,\n",
|
|
" \"learning_rate\": 7.977966217588004e-05,\n",
|
|
" \"loss\": \"kl\",\n",
|
|
" \"model_type\": \"Res2\",\n",
|
|
" \"num_epochs\": 10,\n",
|
|
" \"scheduler_gamma\": 0.5539449021909474\n",
|
|
"}\n",
|
|
"2024-06-03 05:07:36,526 - INFO - Loaded 359668 training images and 39964 test images\n",
|
|
"2024-06-03 06:53:48,871 - INFO - Starting run_62 with hparams {\n",
|
|
" \"batch_size\": 64,\n",
|
|
" \"bin_count\": 16,\n",
|
|
" \"clip_gradients\": true,\n",
|
|
" \"edit_count\": 32,\n",
|
|
" \"learning_rate\": 0.0014725778411180288,\n",
|
|
" \"loss\": \"kl\",\n",
|
|
" \"model_type\": \"SimpleCNN\",\n",
|
|
" \"num_epochs\": 10,\n",
|
|
" \"scheduler_gamma\": 0.981077298963819\n",
|
|
"}\n",
|
|
"2024-06-03 06:53:49,078 - INFO - Loaded 719337 training images and 79927 test images\n",
|
|
"2024-06-03 07:26:40,577 - INFO - Starting run_63 with hparams {\n",
|
|
" \"batch_size\": 32,\n",
|
|
" \"bin_count\": 64,\n",
|
|
" \"clip_gradients\": false,\n",
|
|
" \"edit_count\": 16,\n",
|
|
" \"learning_rate\": 0.0002723042772767375,\n",
|
|
" \"loss\": \"kl\",\n",
|
|
" \"model_type\": \"attention2\",\n",
|
|
" \"num_epochs\": 10,\n",
|
|
" \"scheduler_gamma\": 0.9651950429647194\n",
|
|
"}\n",
|
|
"2024-06-03 07:26:40,602 - INFO - Loaded 359668 training images and 39964 test images\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"from random import choice\n",
|
|
"from itertools import count\n",
|
|
"import json\n",
|
|
"\n",
|
|
"\n",
|
|
"tried = set()\n",
|
|
"\n",
|
|
"for _ in count():\n",
|
|
" current_hyperparameters = {\n",
|
|
" k: v.rvs() if hasattr(v, \"rvs\") else choice(v)\n",
|
|
" for k, v in choice(hyperparameters).items()\n",
|
|
" }\n",
|
|
" key = json.dumps(current_hyperparameters, indent=2, sort_keys=True)\n",
|
|
" if key in tried:\n",
|
|
" continue\n",
|
|
" tried.add(key)\n",
|
|
" logging.info(\n",
|
|
" f\"Starting {get_next_run_name(Path(\"runs\"))} with hparams {key}\"\n",
|
|
" )\n",
|
|
" try:\n",
|
|
" train(current_hyperparameters, max_duration=timedelta(hours=2), use_tqdm=False)\n",
|
|
" except KeyboardInterrupt as e:\n",
|
|
" logging.info(\"Interrupted, stopping\")\n",
|
|
" break\n",
|
|
" except TimeoutError as e:\n",
|
|
" logging.warning(f\"Timeout, aborting experiment\")\n",
|
|
" except Exception as e:\n",
|
|
" logging.error(\n",
|
|
" f\"Error with hparams {current_hyperparameters}:\\n\\t{e}\", stack_info=True\n",
|
|
" )"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "bipolaroid",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.12.2"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|