0.15.14

new pytorch models (opt. scheduler for lr)

Author: bartzbeielstein

pyproject.toml

@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "spotpython"
-version = "0.15.13"
+version = "0.15.14"
 authors = [
   { name="T. Bartz-Beielstein", email="tbb@bartzundbartz.de" }
 ]

src/spotpython/hyperdict/light_hyper_dict.json

@@ -349,11 +349,17 @@
         },
         "optimizer": {
             "levels": [
+                "Adadelta",
+                "Adagrad",
                 "Adam",
                 "AdamW",
+                "SparseAdam",
                 "Adamax",
+                "ASGD",
                 "NAdam",
                 "RAdam",
+                "RMSprop",
+                "Rprop",
                 "SGD"
             ],
             "type": "factor",
@@ -362,8 +368,107 @@
             "class_name": "torch.optim",
             "core_model_parameter_type": "str",
             "lower": 0,
+            "upper": 11
+        },
+        "dropout_prob": {
+            "type": "float",
+            "default": 0.01,
+            "transform": "None",
+            "lower": 0.0,
+            "upper": 0.25
+        },
+        "lr_mult": {
+            "type": "float",
+            "default": 1.0,
+            "transform": "None",
+            "lower": 0.1,
+            "upper": 10.0
+        },
+        "patience": {
+            "type": "int",
+            "default": 2,
+            "transform": "transform_power_2_int",
+            "lower": 2,
+            "upper": 6
+        },
+        "initialization": {
+            "levels": [
+                "Default",
+                "kaiming_uniform",
+                "kaiming_normal",
+                "xavier_uniform",
+                "xavier_normal"
+            ],
+            "type": "factor",
+            "default": "Default",
+            "transform": "None",
+            "core_model_parameter_type": "str",
+            "lower": 0,
+            "upper": 4
+        }
+    },
+    "NNResNetRegressor": {
+        "l1": {
+            "type": "int",
+            "default": 3,
+            "transform": "transform_power_2_int",
+            "lower": 3,
+            "upper": 10
+        },
+        "epochs": {
+            "type": "int",
+            "default": 4,
+            "transform": "transform_power_2_int",
+            "lower": 4,
+            "upper": 9
+        },
+        "batch_size": {
+            "type": "int",
+            "default": 4,
+            "transform": "transform_power_2_int",
+            "lower": 1,
+            "upper": 6
+        },
+        "act_fn": {
+            "levels": [
+                "Sigmoid",
+                "Tanh",
+                "ReLU",
+                "LeakyReLU",
+                "ELU",
+                "Swish"
+            ],
+            "type": "factor",
+            "default": "ReLU",
+            "transform": "None",
+            "class_name": "spotpython.torch.activation",
+            "core_model_parameter_type": "instance()",
+            "lower": 0,
             "upper": 5
         },
+        "optimizer": {
+            "levels": [
+                "Adadelta",
+                "Adagrad",
+                "Adam",
+                "AdamW",
+                "SparseAdam",
+                "Adamax",
+                "ASGD",
+                "NAdam",
+                "RAdam",
+                "RMSprop",
+                "Rprop",
+                "SGD"
+            ],
+            "type": "factor",
+            "default": "SGD",
+            "transform": "None",
+            "class_name": "torch.optim",
+            "core_model_parameter_type": "str",
+            "lower": 0,
+            "upper": 11
+        },
         "dropout_prob": {
             "type": "float",
             "default": 0.01,
@@ -388,15 +493,17 @@
         "initialization": {
             "levels": [
                 "Default",
-                "Kaiming",
-                "Xavier"
+                "kaiming_uniform",
+                "kaiming_normal",
+                "xavier_uniform",
+                "xavier_normal"
             ],
             "type": "factor",
             "default": "Default",
             "transform": "None",
             "core_model_parameter_type": "str",
             "lower": 0,
-            "upper": 2
+            "upper": 4
         }
     }
 }

src/spotpython/light/regression/nn_linear_regressor.py

@@ -3,7 +3,7 @@
 from torch import nn
 from spotpython.hyperparameters.optimizer import optimizer_handler
 import torchmetrics.functional.regression
-from spotpython.utils.math import generate_div2_list
+import torch.optim as optim
 
 
 class NNLinearRegressor(L.LightningModule):
@@ -166,32 +166,57 @@ def __init__(
         if self.hparams.l1 < 4:
             raise ValueError("l1 must be at least 4")
         hidden_sizes = self._get_hidden_sizes()
+
         # Create the network based on the specified hidden sizes
         layers = []
         layer_sizes = [self._L_in] + hidden_sizes
-        layer_size_last = layer_sizes[0]
-        for layer_size in layer_sizes[1:]:
+        for i in range(len(layer_sizes) - 1):
+            current_layer_size = layer_sizes[i]
+            next_layer_size = layer_sizes[i + 1]
             layers += [
-                nn.Linear(layer_size_last, layer_size),
+                nn.Linear(current_layer_size, next_layer_size),
+                nn.BatchNorm1d(next_layer_size),  # Add Batch Normalization here
                 self.hparams.act_fn,
                 nn.Dropout(self.hparams.dropout_prob),
             ]
-            layer_size_last = layer_size
         layers += [nn.Linear(layer_sizes[-1], self._L_out)]
-        # nn.Sequential summarizes a list of modules into a single module, applying them in sequence
-        self.layers = nn.Sequential(*layers)
 
-    def _get_hidden_sizes(self):
-        """
-        Generate the hidden layer sizes for the network.
+        # Wrap the layers into a sequential container
+        self.layers = nn.Sequential(*layers)
 
-        Returns:
-            list: A list of hidden layer sizes.
+        # Initialization (Xavier, Kaiming, or Default)
+        self.apply(self._init_weights)
+
+    def _init_weights(self, module):
+        if isinstance(module, nn.Linear):
+            if self.hparams.initialization == "xavier_uniform":
+                nn.init.xavier_uniform_(module.weight)
+            elif self.hparams.initialization == "xavier_normal":
+                nn.init.xavier_normal_(module.weight)
+            elif self.hparams.initialization == "kaiming_uniform":
+                nn.init.kaiming_uniform_(module.weight)
+            elif self.hparams.initialization == "kaiming_normal":
+                nn.init.kaiming_normal_(module.weight)
+            else:  # "Default"
+                nn.init.uniform_(module.weight)
+            if module.bias is not None:
+                nn.init.zeros_(module.bias)
+
+    def _generate_div2_list(self, n, n_min) -> list:
+        result = []
+        current = n
+        repeats = 1
+        max_repeats = 4
+        while current >= n_min:
+            result.extend([current] * min(repeats, max_repeats))
+            current = current // 2
+            repeats = repeats + 1
+        return result
 
-        """
+    def _get_hidden_sizes(self):
         n_low = self._L_in // 4
         n_high = max(self.hparams.l1, 2 * n_low)
-        hidden_sizes = generate_div2_list(n_high, n_low)
+        hidden_sizes = self._generate_div2_list(n_high, n_low)
         return hidden_sizes
 
     def forward(self, x: torch.Tensor) -> torch.Tensor:
@@ -326,4 +351,18 @@ def configure_optimizers(self) -> torch.optim.Optimizer:
         optimizer = optimizer_handler(
             optimizer_name=self.hparams.optimizer, params=self.parameters(), lr_mult=self.hparams.lr_mult
         )
-        return optimizer
+
+        num_milestones = 3  # Number of milestones to divide the epochs
+        milestones = [int(self.hparams.epochs / (num_milestones + 1) * (i + 1)) for i in range(num_milestones)]
+
+        print(f"Milestones: {milestones}")
+
+        scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=milestones, gamma=0.1)  # Decay factor
+
+        lr_scheduler_config = {
+            "scheduler": scheduler,
+            "interval": "epoch",
+            "frequency": 1,
+        }
+
+        return {"optimizer": optimizer, "lr_scheduler": lr_scheduler_config}