v0.0.62

Author: bartzbeielstein

notebooks/12_spot_hpt_torch_cifar10.ipynb

@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "spotPython"
-version = "0.0.60"
+version = "0.0.62"
 authors = [
   { name="T. Bartz-Beielstein", email="tbb@bartzundbartz.de" }
 ]

notebooks/14_spot_ray_hpt_torch_cifar10.ipynb

@@ -29,7 +29,7 @@
                 "type": "int",
                 "default": 3,
                 "transform": "transform_power_2_int",
-                "lower": 1,
+                "lower": 3,
                 "upper": 4},
         "k_folds": {
             "type": "int",
@@ -68,7 +68,7 @@
                 "type": "int",
                 "default": 3,
                 "transform": "transform_power_2_int",
-                "lower": 1,
+                "lower": 3,
                 "upper": 4},
         "k_folds": {
             "type": "int",

pyproject.toml

@@ -73,16 +73,23 @@ def fun_torch(self, X, fun_control=None):
                 model = self.fun_control["core_model"](**config)
             try:
                 if self.fun_control["eval"] == "train_cv":
-                    df_eval, _ = model.evaluate_cv(dataset=fun_control["train"], shuffle=self.fun_control["shuffle"])
+                    df_eval, _ = model.evaluate_cv(
+                        dataset=fun_control["train"],
+                        shuffle=self.fun_control["shuffle"],
+                        device=self.fun_control["device"],
+                    )
                 elif self.fun_control["eval"] == "test_hold_out":
                     df_eval, _ = model.evaluate_hold_out(
                         dataset=fun_control["train"],
                         shuffle=self.fun_control["shuffle"],
                         test_dataset=fun_control["test"],
+                        device=self.fun_control["device"],
                     )
                 else:  # eval == "train_hold_out"
                     df_eval, _ = model.evaluate_hold_out(
-                        dataset=fun_control["train"], shuffle=self.fun_control["shuffle"]
+                        dataset=fun_control["train"],
+                        shuffle=self.fun_control["shuffle"],
+                        device=self.fun_control["device"],
                     )
             except Exception as err:
                 print(f"Error in fun_torch(). Call to evaluate_model failed. {err=}, {type(err)=}")

src/spotPython/data/torch_hyper_dict.json

@@ -536,7 +536,7 @@ def plot_progress(
         ax = fig.add_subplot(211)
         ax.plot(
             range(1, n_init + 1),
-            s_c[:n_init],
+            s_y[:n_init],
             style[0],
             range(1, n_init + 1),
             [s_c[:n_init].min()] * n_init,

src/spotPython/fun/hypertorch.py

@@ -52,9 +52,9 @@ def validate_fold(self, valloader, criterion, device):
                 val_steps += 1
         return 100.0 * (correct / total)
 
-    def evaluate_cv(self, dataset, shuffle=False):
+    def evaluate_cv(self, dataset, shuffle=False, num_workers=0, device=None):
         try:
-            device = getDevice()
+            device = getDevice(device=device)
             # if torch.cuda.device_count() > 1:
             #     self = nn.DataParallel(self)
             self.to(device)
@@ -65,13 +65,15 @@ def evaluate_cv(self, dataset, shuffle=False):
                 train_subsampler = torch.utils.data.SubsetRandomSampler(train_ids)
                 val_subsampler = torch.utils.data.SubsetRandomSampler(val_ids)
                 trainloader = torch.utils.data.DataLoader(
-                    dataset, batch_size=self.batch_size, sampler=train_subsampler, num_workers=4
+                    dataset, batch_size=self.batch_size, sampler=train_subsampler, num_workers=num_workers
                 )
                 valloader = torch.utils.data.DataLoader(
-                    dataset, batch_size=self.batch_size, sampler=val_subsampler, num_workers=4
+                    dataset, batch_size=self.batch_size, sampler=val_subsampler, num_workers=num_workers
                 )
                 self.reset_weights()
+                # Train fold for several epochs:
                 self.train_fold(trainloader, criterion, optimizer, device)
+                # Validate fold:
                 self.results[fold] = self.validate_fold(valloader, criterion, device)
             df_eval = sum(self.results.values()) / len(self.results.values())
             df_preds = np.nan
@@ -81,11 +83,11 @@ def evaluate_cv(self, dataset, shuffle=False):
             df_preds = np.nan
         return df_eval, df_preds
 
-    def evaluate_hold_out(self, dataset, shuffle, test_dataset=None):
+    def evaluate_hold_out(self, dataset, shuffle, test_dataset=None, device=None):
         lr = self.lr
         epochs = self.epochs
         try:
-            device = getDevice()
+            device = getDevice(device=device)
             self.to(device)
             criterion = nn.CrossEntropyLoss()
             # TODO: optimizer = optim.Adam(self.parameters(), lr=lr)
@@ -99,10 +101,14 @@ def evaluate_hold_out(self, dataset, shuffle, test_dataset=None):
             patience = 5
             best_val_loss = float("inf")
             counter = 0
+            # We only have "one fold" which is trained for several epochs
+            # (we do not have to reset the weights for each fold):
             for epoch in range(epochs):
-                self.train_hold_out(trainloader, criterion, optimizer, device=device, epoch=epoch)
+                print(f"Epoch: {epoch + 1}")
+                # training loss from one epoch:
+                _ = self.train_hold_out(trainloader, criterion, optimizer, device=device)
                 # TODO: scheduler.step()
-                # Early stopping check
+                # Early stopping check. Calculate validation loss from one epoch:
                 val_accuracy, val_loss = self.validate_hold_out(valloader=valloader, criterion=criterion, device=device)
                 if val_loss < best_val_loss:
                     best_val_loss = val_loss
@@ -119,29 +125,30 @@ def evaluate_hold_out(self, dataset, shuffle, test_dataset=None):
             df_eval = np.nan
             df_preds = np.nan
         print(f"Returned to Spot: Validation loss: {df_eval}")
+        print("----------------------------------------------")
         return df_eval, df_preds
 
-    def create_train_val_data_loaders(self, dataset, shuffle):
+    def create_train_val_data_loaders(self, dataset, shuffle, num_workers=0):
         test_abs = int(len(dataset) * 0.6)
         train_subset, val_subset = random_split(dataset, [test_abs, len(dataset) - test_abs])
         trainloader = torch.utils.data.DataLoader(
-            train_subset, batch_size=int(self.batch_size), shuffle=shuffle, num_workers=8, pin_memory=True
+            train_subset, batch_size=int(self.batch_size), shuffle=shuffle, num_workers=num_workers
         )
         valloader = torch.utils.data.DataLoader(
-            val_subset, batch_size=int(self.batch_size), shuffle=shuffle, num_workers=8, pin_memory=True
+            val_subset, batch_size=int(self.batch_size), shuffle=shuffle, num_workers=num_workers
         )
         return trainloader, valloader
 
-    def create_train_test_data_loaders(self, dataset, shuffle, test_dataset):
+    def create_train_test_data_loaders(self, dataset, shuffle, test_dataset, num_workers=0):
         trainloader = torch.utils.data.DataLoader(
-            dataset, batch_size=int(self.batch_size), shuffle=shuffle, num_workers=8, pin_memory=True
+            dataset, batch_size=int(self.batch_size), shuffle=shuffle, num_workers=num_workers
         )
         testloader = torch.utils.data.DataLoader(
-            test_dataset, batch_size=int(self.batch_size), shuffle=shuffle, num_workers=8, pin_memory=True
+            test_dataset, batch_size=int(self.batch_size), shuffle=shuffle, num_workers=num_workers
         )
         return trainloader, testloader
 
-    def train_hold_out(self, trainloader, criterion, optimizer, device, epoch):
+    def train_hold_out(self, trainloader, criterion, optimizer, device):
         running_loss = 0.0
         epoch_steps = 0
         for i, data in enumerate(trainloader, 0):
@@ -158,10 +165,11 @@ def train_hold_out(self, trainloader, criterion, optimizer, device, epoch):
             epoch_steps += 1
             if i % 1000 == 999:  # print every 1000 mini-batches
                 print(
-                    "Epoch: %d, Batch: %5d. Batch Size: %d. Training Loss: %.3f"
-                    % (epoch + 1, i + 1, int(self.batch_size), running_loss / epoch_steps)
+                    "Batch: %5d. Batch Size: %d. Training Loss (running): %.3f"
+                    % (i + 1, int(self.batch_size), running_loss / epoch_steps)
                 )
                 running_loss = 0.0
+        return loss.item()
 
     def validate_hold_out(self, valloader, criterion, device):
         val_loss = 0.0

src/spotPython/spot/spot.py

@@ -1,18 +1,22 @@
 import torch
 
 
-def getDevice():
+def getDevice(device=None):
     """Get cpu, gpu or mps device for training.
+    Args:
+        device (str): Device for training. If None,
+        the device is selected automatically.
     Returns:
         device (str): Device for training.
         Example:
             >>> from spotPython.utils.device import getDevice
             >>> getDevice()
             'cuda:0'
     """
-    device = "cpu"
-    if torch.cuda.is_available():
-        device = "cuda:0"
-    elif torch.backends.mps.is_available():
-        device = "mps"
+    if device is None:
+        device = "cpu"
+        if torch.cuda.is_available():
+            device = "cuda:0"
+        elif torch.backends.mps.is_available():
+            device = "mps"
     return device

src/spotPython/torch/netcore.py

@@ -12,7 +12,8 @@ def fun_control_init():
                 'n_samples': None,
                 'target_column': None,
                 'shuffle': None,
-                'k_folds': None,}
+                'k_folds': None,
+                'device': None}
     """
     fun_control = {
         "data": None,
@@ -23,5 +24,6 @@ def fun_control_init():
         "shuffle": None,
         "eval": None,
         "k_folds": None,
+        "device": None,
     }
     return fun_control