{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": { "metadata": {} }, "outputs": [ { "data": { "text/plain": [ "'Using device cuda:0'" ] }, "execution_count": 1, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import os\n", "from utils import set_up_logging, get_device\n", "from training import train_with_ray_factory\n", "from config import LOGS_PATH, RUNS_PATH, TRAIN_DATA, TEST_DATA\n", "from ray import tune\n", "from ray.tune.schedulers import ASHAScheduler\n", "import os\n", "from ray.air import RunConfig\n", "\n", "set_up_logging(LOGS_PATH)\n", "\n", "TRIAL_COUNT = 100\n", "CHUNK_COUNT = 40\n", "EPOCH_COUNT = 2\n", "\n", "os.environ[\"PYTORCH_CUDA_ALLOC_CONF\"] = \"expandable_segments:True\"\n", "device = get_device()\n", "f\"Using device {device}\"" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "
\n", "
\n", "

Tune Status

\n", " \n", "\n", "\n", "\n", "\n", "\n", "
Current time:2024-09-01 22:06:10
Running for: 00:02:06.64
Memory: 22.4/47.0 GiB
\n", "
\n", "
\n", "
\n", "

System Info

\n", " Using AsyncHyperBand: num_stopped=1
Bracket: Iter 2.000: -5.516964137554169
Logical resource usage: 32.0/32 CPUs, 1.0/1 GPUs (0.0/1.0 accelerator_type:G)\n", "
\n", " \n", "
\n", "
\n", "
\n", "

Trial Status

\n", " \n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "
Trial name status loc batch_size dropout_prob elu_alphafeatures kernel_size leaky_relu_alpha leaky_relu_slope learning_ratemodel_type scheduler_gammause_elu use_residual iter total time (s) chunk_test_loss chunk_training_loss
train_with_ray_b7d3c_00000TERMINATED172.29.235.222:1134109 32 0.00395892 1.35107[8, 16, 32] 5 0.0448715 0.00664526 0.0029226 HistogramNet 0.816752True True 3 70.652 5.22149 56.5472
train_with_ray_b7d3c_00001TERMINATED172.29.235.222:1140440 32 0.0439061 1.74642[8, 8, 8, 8, 8,_c140 3 0.00579656 0.0125715 0.00155182HistogramNet 0.898059True False 2 45.4228 5.79311 64.0481
\n", "
\n", "
\n", "\n" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "\u001b[36m(train_with_ray pid=1134109)\u001b[0m Checkpoint successfully created at: Checkpoint(filesystem=local, path=/home/andras/projects/bipolaroid/runs5/tune/train_with_ray_b7d3c_00000_0_batch_size=32,dropout_prob=0.0040,elu_alpha=1.3511,features=8_16_32,kernel_size=5,leaky_relu_alpha=0._2024-09-01_22-04-03/checkpoint_000000)\n", "\u001b[36m(train_with_ray pid=1134109)\u001b[0m Checkpoint successfully created at: Checkpoint(filesystem=local, path=/home/andras/projects/bipolaroid/runs5/tune/train_with_ray_b7d3c_00000_0_batch_size=32,dropout_prob=0.0040,elu_alpha=1.3511,features=8_16_32,kernel_size=5,leaky_relu_alpha=0._2024-09-01_22-04-03/checkpoint_000001)\n", "\u001b[36m(train_with_ray pid=1134109)\u001b[0m Checkpoint successfully created at: Checkpoint(filesystem=local, path=/home/andras/projects/bipolaroid/runs5/tune/train_with_ray_b7d3c_00000_0_batch_size=32,dropout_prob=0.0040,elu_alpha=1.3511,features=8_16_32,kernel_size=5,leaky_relu_alpha=0._2024-09-01_22-04-03/checkpoint_000002)\n", "\u001b[36m(train_with_ray pid=1140440)\u001b[0m Checkpoint successfully created at: Checkpoint(filesystem=local, path=/home/andras/projects/bipolaroid/runs5/tune/train_with_ray_b7d3c_00001_1_batch_size=32,dropout_prob=0.0439,elu_alpha=1.7464,features=8_8_8_8_8_8_8,kernel_size=3,leaky_relu_al_2024-09-01_22-04-03/checkpoint_000000)\n", "2024-09-01 22:06:10,132\tINFO tune.py:1009 -- Wrote the latest version of all result files and experiment state to '/home/andras/projects/bipolaroid/runs5/tune' in 0.0020s.\n", "2024-09-01 22:06:10,135\tINFO tune.py:1041 -- Total run time: 126.67 seconds (126.64 seconds for the tuning loop).\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\u001b[36m(train_with_ray pid=1140440)\u001b[0m Checkpoint successfully created at: Checkpoint(filesystem=local, path=/home/andras/projects/bipolaroid/runs5/tune/train_with_ray_b7d3c_00001_1_batch_size=32,dropout_prob=0.0439,elu_alpha=1.7464,features=8_8_8_8_8_8_8,kernel_size=3,leaky_relu_al_2024-09-01_22-04-03/checkpoint_000001)\n" ] } ], "source": [ "config = {\n", " \"batch_size\": 32,\n", " \"edit_count\": EPOCH_COUNT,\n", " \"bin_count\": 32,\n", " \"learning_rate\": tune.loguniform(5e-4, 5e-3),\n", " \"scheduler_gamma\": tune.uniform(0.8, 0.95),\n", " \"elu_alpha\": tune.uniform(0.5, 2),\n", " \"leaky_relu_slope\": tune.uniform(0, 0.03),\n", " \"dropout_prob\": tune.uniform(0, 0.1),\n", " \"chunk_count\": CHUNK_COUNT,\n", " \"features\": tune.choice(\n", " [\n", " [16, 32, 64],\n", " [16, 32, 64, 128],\n", " [32, 64],\n", " [32, 128],\n", " [8, 16, 32],\n", " [8, 8, 8, 8, 8],\n", " [8, 8, 8, 8, 8, 8, 8],\n", " [16, 16, 16],\n", " [16, 16, 16, 16, 16],\n", " [32, 32, 32],\n", " [32, 32, 32, 32],\n", " [64, 64],\n", " [64, 64, 64],\n", " ]\n", " ),\n", " \"use_residual\": tune.choice([True, False]),\n", " \"kernel_size\": tune.choice([3, 5]),\n", " \"model_type\": tune.choice([\"HistogramNet\"]),\n", " \"use_instance_norm\": True,\n", " \"use_elu\": tune.choice([True, False]),\n", " \"leaky_relu_alpha\": tune.uniform(0, 0.05),\n", "}\n", "scheduler = ASHAScheduler(max_t=config[\"chunk_count\"], grace_period=2)\n", "\n", "tuner = tune.Tuner(\n", " tune.with_resources(\n", " tune.with_parameters(\n", " train_with_ray_factory(\n", " train_data_paths=TRAIN_DATA,\n", " test_data_paths=TEST_DATA,\n", " device=device,\n", " log_dir=RUNS_PATH / \"custom\",\n", " )\n", " ),\n", " resources={\"cpu\": 32, \"gpu\": 1},\n", " ),\n", " run_config=RunConfig(storage_path=RUNS_PATH, name=\"tune\"),\n", " tune_config=tune.TuneConfig(\n", " metric=\"chunk_test_loss\",\n", " mode=\"min\",\n", " scheduler=scheduler,\n", " num_samples=TRIAL_COUNT,\n", " ),\n", " param_space=config,\n", ")\n", "results = tuner.fit()" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Best trial config: {'batch_size': 32, 'edit_count': 3, 'bin_count': 32, 'learning_rate': 0.0029226033808016005, 'scheduler_gamma': 0.8167516482513361, 'elu_alpha': 1.3510723758569865, 'leaky_relu_slope': 0.0066452562138349025, 'dropout_prob': 0.0039589213934103865, 'chunk_count': 4, 'features': [8, 16, 32], 'use_residual': True, 'kernel_size': 5, 'model_type': 'HistogramNet', 'use_instance_norm': True, 'use_elu': True, 'leaky_relu_alpha': 0.04487148648446764}\n", "Best trial final validation loss: 5.2214884757995605\n" ] } ], "source": [ "best_result = results.get_best_result(\"chunk_test_loss\", \"min\")\n", "\n", "print(\"Best trial config: {}\".format(best_result.config))\n", "print(\n", " \"Best trial final validation loss: {}\".format(\n", " best_result.metrics[\"chunk_test_loss\"]\n", " )\n", ")" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "# hparams = {\n", "# \"batch_size\": 64,\n", "# \"edit_count\": 12,\n", "# \"bin_count\": 16,\n", "# \"learning_rate\": 0.0006126108207352808,\n", "# \"scheduler_gamma\": 0.9382286228762693,\n", "# \"num_epochs\": 10,\n", "# \"elu_alpha\": 1.3092260477215776,\n", "# \"leaky_relu_slope\": 0.029438156325552762,\n", "# \"dropout_prob\": 0.06261255195786307,\n", "# \"features\": [8, 16, 32],\n", "# \"use_residual\": True,\n", "# \"kernel_size\": 5,\n", "# \"model_type\": \"HistogramNet\",\n", "# \"use_instance_norm\": True,\n", "# \"use_elu\": False,\n", "# \"leaky_relu_alpha\": 0.03745605986732464,\n", "# }\n", "\n", "# train(\n", "# hparams,\n", "# train_data_paths=TRAIN_DATA,\n", "# test_data_paths=TEST_DATA,\n", "# log_dir=RUNS_PATH,\n", "# max_duration=None,\n", "# use_tqdm=True,\n", "# device=device,\n", "# **hparams\n", "# )" ] } ], "metadata": { "kernelspec": { "display_name": "bipolaroid", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.2" } }, "nbformat": 4, "nbformat_minor": 2 }