Skip to content

Commit 3043232

Browse files
bartzbeielsteinbartzbeielstein
committed
tests optimizers
1 parent 5981890 commit 3043232

5 files changed

Lines changed: 145 additions & 58 deletions

File tree

‎notebooks/00_spotPython_tests.ipynb‎

Lines changed: 74 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -790,7 +790,46 @@
790790
},
791791
{
792792
"cell_type": "code",
793-
"execution_count": 20,
793+
"execution_count": null,
794+
"metadata": {},
795+
"outputs": [],
796+
"source": [
797+
"from torch.utils.data import DataLoader\n",
798+
"from spotPython.data.diabetes import Diabetes\n",
799+
"from spotPython.light.netlightregression import NetLightRegression\n",
800+
"from torch import nn\n",
801+
"import lightning as L\n",
802+
"PATH_DATASETS = './data'\n",
803+
"BATCH_SIZE = 8\n",
804+
"\n",
805+
"dataset = Diabetes()\n",
806+
"train_loader = DataLoader(dataset, batch_size=BATCH_SIZE)\n",
807+
"test_loader = DataLoader(dataset, batch_size=BATCH_SIZE)\n",
808+
"val_loader = DataLoader(dataset, batch_size=BATCH_SIZE)\n",
809+
"batch_x, batch_y = next(iter(train_loader)) \n",
810+
"print(batch_x.shape)\n",
811+
"print(batch_y.shape)\n",
812+
"\n",
813+
"net_light_base = NetLightRegression(l1=128, epochs=10, batch_size=BATCH_SIZE,\n",
814+
" initialization='xavier', act_fn=nn.ReLU(),\n",
815+
" optimizer='Adam', dropout_prob=0.1, lr_mult=0.1,\n",
816+
" patience=5, _L_in=10, _L_out=1)\n",
817+
"trainer = L.Trainer(max_epochs=2, enable_progress_bar=False)\n",
818+
"trainer.fit(net_light_base, train_loader)\n",
819+
"trainer.validate(net_light_base, val_loader)\n",
820+
"trainer.test(net_light_base, test_loader)\n"
821+
]
822+
},
823+
{
824+
"cell_type": "markdown",
825+
"metadata": {},
826+
"source": [
827+
"# tests optimizer_handler"
828+
]
829+
},
830+
{
831+
"cell_type": "code",
832+
"execution_count": 10,
794833
"metadata": {},
795834
"outputs": [
796835
{
@@ -811,52 +850,32 @@
811850
"15.9 K Trainable params\n",
812851
"0 Non-trainable params\n",
813852
"15.9 K Total params\n",
814-
"0.064 Total estimated model params size (MB)\n"
815-
]
816-
},
817-
{
818-
"name": "stdout",
819-
"output_type": "stream",
820-
"text": [
821-
"torch.Size([8, 10])\n",
822-
"torch.Size([8])\n"
823-
]
824-
},
825-
{
826-
"name": "stderr",
827-
"output_type": "stream",
828-
"text": [
853+
"0.064 Total estimated model params size (MB)\n",
829854
"/Users/bartz/miniforge3/envs/spotCondaEnv/lib/python3.11/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:441: The 'train_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=11` in the `DataLoader` to improve performance.\n",
830855
"`Trainer.fit` stopped: `max_epochs=2` reached.\n",
831-
"/Users/bartz/miniforge3/envs/spotCondaEnv/lib/python3.11/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:441: The 'val_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=11` in the `DataLoader` to improve performance.\n",
832-
"/Users/bartz/miniforge3/envs/spotCondaEnv/lib/python3.11/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:441: The 'test_dataloader' does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` to `num_workers=11` in the `DataLoader` to improve performance.\n"
833-
]
834-
},
835-
{
836-
"name": "stdout",
837-
"output_type": "stream",
838-
"text": [
839-
"────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n",
840-
" Validate metric DataLoader 0\n",
841-
"────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n",
842-
" hp_metric 28981.529296875\n",
843-
" val_loss 28981.529296875\n",
844-
"────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n",
845-
"────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n",
846-
" Test metric DataLoader 0\n",
847-
"────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n",
848-
" hp_metric 28981.529296875\n",
849-
" val_loss 28981.529296875\n",
850-
"────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\n"
856+
"GPU available: True (mps), used: True\n",
857+
"TPU available: False, using: 0 TPU cores\n",
858+
"IPU available: False, using: 0 IPUs\n",
859+
"HPU available: False, using: 0 HPUs\n",
860+
"\n",
861+
" | Name | Type | Params | In sizes | Out sizes\n",
862+
"-------------------------------------------------------------\n",
863+
"0 | layers | Sequential | 15.9 K | [8, 10] | [8, 1] \n",
864+
"-------------------------------------------------------------\n",
865+
"15.9 K Trainable params\n",
866+
"0 Non-trainable params\n",
867+
"15.9 K Total params\n",
868+
"0.064 Total estimated model params size (MB)\n",
869+
"`Trainer.fit` stopped: `max_epochs=2` reached.\n"
851870
]
852871
},
853872
{
854873
"data": {
855874
"text/plain": [
856-
"[{'val_loss': 28981.529296875, 'hp_metric': 28981.529296875}]"
875+
"True"
857876
]
858877
},
859-
"execution_count": 20,
878+
"execution_count": 10,
860879
"metadata": {},
861880
"output_type": "execute_result"
862881
}
@@ -867,25 +886,35 @@
867886
"from spotPython.light.netlightregression import NetLightRegression\n",
868887
"from torch import nn\n",
869888
"import lightning as L\n",
870-
"PATH_DATASETS = './data'\n",
889+
"\n",
871890
"BATCH_SIZE = 8\n",
891+
"lr_mult=0.1\n",
872892
"\n",
873893
"dataset = Diabetes()\n",
874894
"train_loader = DataLoader(dataset, batch_size=BATCH_SIZE)\n",
875895
"test_loader = DataLoader(dataset, batch_size=BATCH_SIZE)\n",
876896
"val_loader = DataLoader(dataset, batch_size=BATCH_SIZE)\n",
877-
"batch_x, batch_y = next(iter(train_loader)) \n",
878-
"print(batch_x.shape)\n",
879-
"print(batch_y.shape)\n",
880897
"\n",
881898
"net_light_base = NetLightRegression(l1=128, epochs=10, batch_size=BATCH_SIZE,\n",
882899
" initialization='xavier', act_fn=nn.ReLU(),\n",
883-
" optimizer='Adam', dropout_prob=0.1, lr_mult=0.1,\n",
900+
" optimizer='Adam', dropout_prob=0.1, lr_mult=lr_mult,\n",
884901
" patience=5, _L_in=10, _L_out=1)\n",
885902
"trainer = L.Trainer(max_epochs=2, enable_progress_bar=False)\n",
886903
"trainer.fit(net_light_base, train_loader)\n",
887-
"trainer.validate(net_light_base, val_loader)\n",
888-
"trainer.test(net_light_base, test_loader)\n"
904+
"# Adam uses a lr which is calculated as lr=lr_mult * 0.001, so this value\n",
905+
"# should be 0.1 * 0.001 = 0.0001 \n",
906+
"trainer.optimizers[0].param_groups[0][\"lr\"] == lr_mult*0.001\n",
907+
"\n",
908+
"\n",
909+
"net_light_base = NetLightRegression(l1=128, epochs=10, batch_size=BATCH_SIZE,\n",
910+
" initialization='xavier', act_fn=nn.ReLU(),\n",
911+
" optimizer='Adadelta', dropout_prob=0.1, lr_mult=lr_mult,\n",
912+
" patience=5, _L_in=10, _L_out=1)\n",
913+
"trainer = L.Trainer(max_epochs=2, enable_progress_bar=False)\n",
914+
"trainer.fit(net_light_base, train_loader)\n",
915+
"# Adadelta uses a lr which is calculated as lr=lr_mult * 1.0, so this value\n",
916+
"# should be 1.0 * 0.1 = 0.1 \n",
917+
"trainer.optimizers[0].param_groups[0][\"lr\"] == lr_mult*1.0\n"
889918
]
890919
},
891920
{

‎pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
77

88
[project]
99
name = "spotPython"
10-
version = "0.6.55"
10+
version = "0.6.56"
1111
authors = [
1212
{ name="T. Bartz-Beielstein", email="tbb@bartzundbartz.de" }
1313
]

‎src/spotPython/hyperparameters/optimizer.py‎

Lines changed: 31 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -39,17 +39,37 @@ def optimizer_handler(
3939
An instance of the specified optimizer.
4040
4141
Examples:
42-
>>>
43-
>>> model = torch.nn.Linear(10, 1)
44-
>>> optimizer = optimizer_handler("Adadelta", model.parameters(), lr_mult=0.5)
45-
>>> print(optimizer)
46-
Adadelta (
47-
Parameter Group 0
48-
eps: 1e-06
49-
lr: 0.5
50-
rho: 0.9
51-
weight_decay: 0
52-
)
42+
from torch.utils.data import DataLoader
43+
from spotPython.data.diabetes import Diabetes
44+
from spotPython.light.netlightregression import NetLightRegression
45+
from torch import nn
46+
import lightning as L
47+
BATCH_SIZE = 8
48+
lr_mult=0.1
49+
dataset = Diabetes()
50+
train_loader = DataLoader(dataset, batch_size=BATCH_SIZE)
51+
test_loader = DataLoader(dataset, batch_size=BATCH_SIZE)
52+
val_loader = DataLoader(dataset, batch_size=BATCH_SIZE)
53+
# First example: Adam
54+
net_light_base = NetLightRegression(l1=128, epochs=10, batch_size=BATCH_SIZE,
55+
initialization='xavier', act_fn=nn.ReLU(),
56+
optimizer='Adam', dropout_prob=0.1, lr_mult=lr_mult,
57+
patience=5, _L_in=10, _L_out=1)
58+
trainer = L.Trainer(max_epochs=2, enable_progress_bar=False)
59+
trainer.fit(net_light_base, train_loader)
60+
# Adam uses a lr which is calculated as lr=lr_mult * 0.001, so this value
61+
# should be 0.1 * 0.001 = 0.0001
62+
trainer.optimizers[0].param_groups[0]["lr"] == lr_mult*0.001
63+
# Second example: Adadelta
64+
net_light_base = NetLightRegression(l1=128, epochs=10, batch_size=BATCH_SIZE,
65+
initialization='xavier', act_fn=nn.ReLU(),
66+
optimizer='Adadelta', dropout_prob=0.1, lr_mult=lr_mult,
67+
patience=5, _L_in=10, _L_out=1)
68+
trainer = L.Trainer(max_epochs=2, enable_progress_bar=False)
69+
trainer.fit(net_light_base, train_loader)
70+
# Adadelta uses a lr which is calculated as lr=lr_mult * 1.0, so this value
71+
# should be 1.0 * 0.1 = 0.1
72+
trainer.optimizers[0].param_groups[0]["lr"] == lr_mult*1.0
5373
"""
5474
if optimizer_name == "Adadelta":
5575
return torch.optim.Adadelta(

‎src/spotPython/light/traintest.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
import lightning as L
22

3-
# from spotPython.light.csvdatamodule import CSVDataModule
43
from spotPython.data.lightdatamodule import LightDataModule
54
from spotPython.data.lightcrossvalidationdatamodule import LightCrossValidationDataModule
65
from spotPython.utils.eda import generate_config_id

‎test/test_optimizer_handler.py‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
import pytest
2+
from torch.utils.data import DataLoader
3+
from spotPython.data.diabetes import Diabetes
4+
from spotPython.light.netlightregression import NetLightRegression
5+
from torch import nn
6+
import lightning as L
7+
8+
9+
def test_optimizer_handler():
10+
BATCH_SIZE = 8
11+
lr_mult=0.1
12+
13+
dataset = Diabetes()
14+
train_loader = DataLoader(dataset, batch_size=BATCH_SIZE)
15+
16+
net_light_base = NetLightRegression(l1=128, epochs=10, batch_size=BATCH_SIZE,
17+
initialization='xavier', act_fn=nn.ReLU(),
18+
optimizer='Adam', dropout_prob=0.1, lr_mult=lr_mult,
19+
patience=5, _L_in=10, _L_out=1)
20+
trainer = L.Trainer(max_epochs=2, enable_progress_bar=False)
21+
trainer.fit(net_light_base, train_loader)
22+
# Adam uses a lr which is calculated as lr=lr_mult * 0.001, so this value
23+
# should be 0.1 * 0.001 = 0.0001
24+
assert(trainer.optimizers[0].param_groups[0]["lr"] == lr_mult*0.001)
25+
26+
27+
net_light_base = NetLightRegression(l1=128, epochs=10, batch_size=BATCH_SIZE,
28+
initialization='xavier', act_fn=nn.ReLU(),
29+
optimizer='Adadelta', dropout_prob=0.1, lr_mult=lr_mult,
30+
patience=5, _L_in=10, _L_out=1)
31+
trainer = L.Trainer(max_epochs=2, enable_progress_bar=False)
32+
trainer.fit(net_light_base, train_loader)
33+
# Adadelta uses a lr which is calculated as lr=lr_mult * 1.0, so this value
34+
# should be 1.0 * 0.1 = 0.1
35+
assert(trainer.optimizers[0].param_groups[0]["lr"] == lr_mult*1.0)
36+
37+
38+
if __name__ == "__main__":
39+
pytest.main(["-v", __file__])

0 commit comments

Comments
 (0)