286 lines
13 KiB
Text
286 lines
13 KiB
Text
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 1,
|
|
"metadata": {
|
|
"metadata": {}
|
|
},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/plain": [
|
|
"'Using device cuda:0'"
|
|
]
|
|
},
|
|
"execution_count": 1,
|
|
"metadata": {},
|
|
"output_type": "execute_result"
|
|
}
|
|
],
|
|
"source": [
|
|
"import os\n",
|
|
"from utils import set_up_logging, get_device\n",
|
|
"from training import train_with_ray_factory\n",
|
|
"from config import LOGS_PATH, RUNS_PATH, TRAIN_DATA, TEST_DATA\n",
|
|
"from ray import tune\n",
|
|
"from ray.tune.schedulers import ASHAScheduler\n",
|
|
"import os\n",
|
|
"from ray.air import RunConfig\n",
|
|
"\n",
|
|
"set_up_logging(LOGS_PATH)\n",
|
|
"\n",
|
|
"TRIAL_COUNT = 100\n",
|
|
"CHUNK_COUNT = 40\n",
|
|
"EPOCH_COUNT = 2\n",
|
|
"\n",
|
|
"os.environ[\"PYTORCH_CUDA_ALLOC_CONF\"] = \"expandable_segments:True\"\n",
|
|
"device = get_device()\n",
|
|
"f\"Using device {device}\""
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 2,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"data": {
|
|
"text/html": [
|
|
"<div class=\"tuneStatus\">\n",
|
|
" <div style=\"display: flex;flex-direction: row\">\n",
|
|
" <div style=\"display: flex;flex-direction: column;\">\n",
|
|
" <h3>Tune Status</h3>\n",
|
|
" <table>\n",
|
|
"<tbody>\n",
|
|
"<tr><td>Current time:</td><td>2024-09-01 22:06:10</td></tr>\n",
|
|
"<tr><td>Running for: </td><td>00:02:06.64 </td></tr>\n",
|
|
"<tr><td>Memory: </td><td>22.4/47.0 GiB </td></tr>\n",
|
|
"</tbody>\n",
|
|
"</table>\n",
|
|
" </div>\n",
|
|
" <div class=\"vDivider\"></div>\n",
|
|
" <div class=\"systemInfo\">\n",
|
|
" <h3>System Info</h3>\n",
|
|
" Using AsyncHyperBand: num_stopped=1<br>Bracket: Iter 2.000: -5.516964137554169<br>Logical resource usage: 32.0/32 CPUs, 1.0/1 GPUs (0.0/1.0 accelerator_type:G)\n",
|
|
" </div>\n",
|
|
" \n",
|
|
" </div>\n",
|
|
" <div class=\"hDivider\"></div>\n",
|
|
" <div class=\"trialStatus\">\n",
|
|
" <h3>Trial Status</h3>\n",
|
|
" <table>\n",
|
|
"<thead>\n",
|
|
"<tr><th>Trial name </th><th>status </th><th>loc </th><th style=\"text-align: right;\"> batch_size</th><th style=\"text-align: right;\"> dropout_prob</th><th style=\"text-align: right;\"> elu_alpha</th><th>features </th><th style=\"text-align: right;\"> kernel_size</th><th style=\"text-align: right;\"> leaky_relu_alpha</th><th style=\"text-align: right;\"> leaky_relu_slope</th><th style=\"text-align: right;\"> learning_rate</th><th>model_type </th><th style=\"text-align: right;\"> scheduler_gamma</th><th>use_elu </th><th>use_residual </th><th style=\"text-align: right;\"> iter</th><th style=\"text-align: right;\"> total time (s)</th><th style=\"text-align: right;\"> chunk_test_loss</th><th style=\"text-align: right;\"> chunk_training_loss</th></tr>\n",
|
|
"</thead>\n",
|
|
"<tbody>\n",
|
|
"<tr><td>train_with_ray_b7d3c_00000</td><td>TERMINATED</td><td>172.29.235.222:1134109</td><td style=\"text-align: right;\"> 32</td><td style=\"text-align: right;\"> 0.00395892</td><td style=\"text-align: right;\"> 1.35107</td><td>[8, 16, 32] </td><td style=\"text-align: right;\"> 5</td><td style=\"text-align: right;\"> 0.0448715 </td><td style=\"text-align: right;\"> 0.00664526</td><td style=\"text-align: right;\"> 0.0029226 </td><td>HistogramNet</td><td style=\"text-align: right;\"> 0.816752</td><td>True </td><td>True </td><td style=\"text-align: right;\"> 3</td><td style=\"text-align: right;\"> 70.652 </td><td style=\"text-align: right;\"> 5.22149</td><td style=\"text-align: right;\"> 56.5472</td></tr>\n",
|
|
"<tr><td>train_with_ray_b7d3c_00001</td><td>TERMINATED</td><td>172.29.235.222:1140440</td><td style=\"text-align: right;\"> 32</td><td style=\"text-align: right;\"> 0.0439061 </td><td style=\"text-align: right;\"> 1.74642</td><td>[8, 8, 8, 8, 8,_c140</td><td style=\"text-align: right;\"> 3</td><td style=\"text-align: right;\"> 0.00579656</td><td style=\"text-align: right;\"> 0.0125715 </td><td style=\"text-align: right;\"> 0.00155182</td><td>HistogramNet</td><td style=\"text-align: right;\"> 0.898059</td><td>True </td><td>False </td><td style=\"text-align: right;\"> 2</td><td style=\"text-align: right;\"> 45.4228</td><td style=\"text-align: right;\"> 5.79311</td><td style=\"text-align: right;\"> 64.0481</td></tr>\n",
|
|
"</tbody>\n",
|
|
"</table>\n",
|
|
" </div>\n",
|
|
"</div>\n",
|
|
"<style>\n",
|
|
".tuneStatus {\n",
|
|
" color: var(--jp-ui-font-color1);\n",
|
|
"}\n",
|
|
".tuneStatus .systemInfo {\n",
|
|
" display: flex;\n",
|
|
" flex-direction: column;\n",
|
|
"}\n",
|
|
".tuneStatus td {\n",
|
|
" white-space: nowrap;\n",
|
|
"}\n",
|
|
".tuneStatus .trialStatus {\n",
|
|
" display: flex;\n",
|
|
" flex-direction: column;\n",
|
|
"}\n",
|
|
".tuneStatus h3 {\n",
|
|
" font-weight: bold;\n",
|
|
"}\n",
|
|
".tuneStatus .hDivider {\n",
|
|
" border-bottom-width: var(--jp-border-width);\n",
|
|
" border-bottom-color: var(--jp-border-color0);\n",
|
|
" border-bottom-style: solid;\n",
|
|
"}\n",
|
|
".tuneStatus .vDivider {\n",
|
|
" border-left-width: var(--jp-border-width);\n",
|
|
" border-left-color: var(--jp-border-color0);\n",
|
|
" border-left-style: solid;\n",
|
|
" margin: 0.5em 1em 0.5em 1em;\n",
|
|
"}\n",
|
|
"</style>\n"
|
|
],
|
|
"text/plain": [
|
|
"<IPython.core.display.HTML object>"
|
|
]
|
|
},
|
|
"metadata": {},
|
|
"output_type": "display_data"
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\u001b[36m(train_with_ray pid=1134109)\u001b[0m Checkpoint successfully created at: Checkpoint(filesystem=local, path=/home/andras/projects/bipolaroid/runs5/tune/train_with_ray_b7d3c_00000_0_batch_size=32,dropout_prob=0.0040,elu_alpha=1.3511,features=8_16_32,kernel_size=5,leaky_relu_alpha=0._2024-09-01_22-04-03/checkpoint_000000)\n",
|
|
"\u001b[36m(train_with_ray pid=1134109)\u001b[0m Checkpoint successfully created at: Checkpoint(filesystem=local, path=/home/andras/projects/bipolaroid/runs5/tune/train_with_ray_b7d3c_00000_0_batch_size=32,dropout_prob=0.0040,elu_alpha=1.3511,features=8_16_32,kernel_size=5,leaky_relu_alpha=0._2024-09-01_22-04-03/checkpoint_000001)\n",
|
|
"\u001b[36m(train_with_ray pid=1134109)\u001b[0m Checkpoint successfully created at: Checkpoint(filesystem=local, path=/home/andras/projects/bipolaroid/runs5/tune/train_with_ray_b7d3c_00000_0_batch_size=32,dropout_prob=0.0040,elu_alpha=1.3511,features=8_16_32,kernel_size=5,leaky_relu_alpha=0._2024-09-01_22-04-03/checkpoint_000002)\n",
|
|
"\u001b[36m(train_with_ray pid=1140440)\u001b[0m Checkpoint successfully created at: Checkpoint(filesystem=local, path=/home/andras/projects/bipolaroid/runs5/tune/train_with_ray_b7d3c_00001_1_batch_size=32,dropout_prob=0.0439,elu_alpha=1.7464,features=8_8_8_8_8_8_8,kernel_size=3,leaky_relu_al_2024-09-01_22-04-03/checkpoint_000000)\n",
|
|
"2024-09-01 22:06:10,132\tINFO tune.py:1009 -- Wrote the latest version of all result files and experiment state to '/home/andras/projects/bipolaroid/runs5/tune' in 0.0020s.\n",
|
|
"2024-09-01 22:06:10,135\tINFO tune.py:1041 -- Total run time: 126.67 seconds (126.64 seconds for the tuning loop).\n"
|
|
]
|
|
},
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"\u001b[36m(train_with_ray pid=1140440)\u001b[0m Checkpoint successfully created at: Checkpoint(filesystem=local, path=/home/andras/projects/bipolaroid/runs5/tune/train_with_ray_b7d3c_00001_1_batch_size=32,dropout_prob=0.0439,elu_alpha=1.7464,features=8_8_8_8_8_8_8,kernel_size=3,leaky_relu_al_2024-09-01_22-04-03/checkpoint_000001)\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"config = {\n",
|
|
" \"batch_size\": 32,\n",
|
|
" \"edit_count\": EPOCH_COUNT,\n",
|
|
" \"bin_count\": 32,\n",
|
|
" \"learning_rate\": tune.loguniform(5e-4, 5e-3),\n",
|
|
" \"scheduler_gamma\": tune.uniform(0.8, 0.95),\n",
|
|
" \"elu_alpha\": tune.uniform(0.5, 2),\n",
|
|
" \"leaky_relu_slope\": tune.uniform(0, 0.03),\n",
|
|
" \"dropout_prob\": tune.uniform(0, 0.1),\n",
|
|
" \"chunk_count\": CHUNK_COUNT,\n",
|
|
" \"features\": tune.choice(\n",
|
|
" [\n",
|
|
" [16, 32, 64],\n",
|
|
" [16, 32, 64, 128],\n",
|
|
" [32, 64],\n",
|
|
" [32, 128],\n",
|
|
" [8, 16, 32],\n",
|
|
" [8, 8, 8, 8, 8],\n",
|
|
" [8, 8, 8, 8, 8, 8, 8],\n",
|
|
" [16, 16, 16],\n",
|
|
" [16, 16, 16, 16, 16],\n",
|
|
" [32, 32, 32],\n",
|
|
" [32, 32, 32, 32],\n",
|
|
" [64, 64],\n",
|
|
" [64, 64, 64],\n",
|
|
" ]\n",
|
|
" ),\n",
|
|
" \"use_residual\": tune.choice([True, False]),\n",
|
|
" \"kernel_size\": tune.choice([3, 5]),\n",
|
|
" \"model_type\": tune.choice([\"HistogramNet\"]),\n",
|
|
" \"use_instance_norm\": True,\n",
|
|
" \"use_elu\": tune.choice([True, False]),\n",
|
|
" \"leaky_relu_alpha\": tune.uniform(0, 0.05),\n",
|
|
"}\n",
|
|
"scheduler = ASHAScheduler(max_t=config[\"chunk_count\"], grace_period=2)\n",
|
|
"\n",
|
|
"tuner = tune.Tuner(\n",
|
|
" tune.with_resources(\n",
|
|
" tune.with_parameters(\n",
|
|
" train_with_ray_factory(\n",
|
|
" train_data_paths=TRAIN_DATA,\n",
|
|
" test_data_paths=TEST_DATA,\n",
|
|
" device=device,\n",
|
|
" log_dir=RUNS_PATH / \"custom\",\n",
|
|
" )\n",
|
|
" ),\n",
|
|
" resources={\"cpu\": 32, \"gpu\": 1},\n",
|
|
" ),\n",
|
|
" run_config=RunConfig(storage_path=RUNS_PATH, name=\"tune\"),\n",
|
|
" tune_config=tune.TuneConfig(\n",
|
|
" metric=\"chunk_test_loss\",\n",
|
|
" mode=\"min\",\n",
|
|
" scheduler=scheduler,\n",
|
|
" num_samples=TRIAL_COUNT,\n",
|
|
" ),\n",
|
|
" param_space=config,\n",
|
|
")\n",
|
|
"results = tuner.fit()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"Best trial config: {'batch_size': 32, 'edit_count': 3, 'bin_count': 32, 'learning_rate': 0.0029226033808016005, 'scheduler_gamma': 0.8167516482513361, 'elu_alpha': 1.3510723758569865, 'leaky_relu_slope': 0.0066452562138349025, 'dropout_prob': 0.0039589213934103865, 'chunk_count': 4, 'features': [8, 16, 32], 'use_residual': True, 'kernel_size': 5, 'model_type': 'HistogramNet', 'use_instance_norm': True, 'use_elu': True, 'leaky_relu_alpha': 0.04487148648446764}\n",
|
|
"Best trial final validation loss: 5.2214884757995605\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"best_result = results.get_best_result(\"chunk_test_loss\", \"min\")\n",
|
|
"\n",
|
|
"print(\"Best trial config: {}\".format(best_result.config))\n",
|
|
"print(\n",
|
|
" \"Best trial final validation loss: {}\".format(\n",
|
|
" best_result.metrics[\"chunk_test_loss\"]\n",
|
|
" )\n",
|
|
")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# hparams = {\n",
|
|
"# \"batch_size\": 64,\n",
|
|
"# \"edit_count\": 12,\n",
|
|
"# \"bin_count\": 16,\n",
|
|
"# \"learning_rate\": 0.0006126108207352808,\n",
|
|
"# \"scheduler_gamma\": 0.9382286228762693,\n",
|
|
"# \"num_epochs\": 10,\n",
|
|
"# \"elu_alpha\": 1.3092260477215776,\n",
|
|
"# \"leaky_relu_slope\": 0.029438156325552762,\n",
|
|
"# \"dropout_prob\": 0.06261255195786307,\n",
|
|
"# \"features\": [8, 16, 32],\n",
|
|
"# \"use_residual\": True,\n",
|
|
"# \"kernel_size\": 5,\n",
|
|
"# \"model_type\": \"HistogramNet\",\n",
|
|
"# \"use_instance_norm\": True,\n",
|
|
"# \"use_elu\": False,\n",
|
|
"# \"leaky_relu_alpha\": 0.03745605986732464,\n",
|
|
"# }\n",
|
|
"\n",
|
|
"# train(\n",
|
|
"# hparams,\n",
|
|
"# train_data_paths=TRAIN_DATA,\n",
|
|
"# test_data_paths=TEST_DATA,\n",
|
|
"# log_dir=RUNS_PATH,\n",
|
|
"# max_duration=None,\n",
|
|
"# use_tqdm=True,\n",
|
|
"# device=device,\n",
|
|
"# **hparams\n",
|
|
"# )"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "bipolaroid",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.12.2"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|