Merge pull request #84 from CCPBioSim/81-verbose-YAML-bug

Ensure verbose flag in YAML sets logging to DEBUG mode correctly

Author: harryswift01

CodeEntropy/config/arg_config_manager.py

@@ -48,18 +48,17 @@
         "help": "Enable verbose output",
     },
     "thread": {"type": int, "help": "How many multiprocess to use", "default": 1},
-    "outfile": {
+    "output_file": {
         "type": str,
         "help": "Name of the file where the output will be written",
-        "default": "outfile.json",
-    },
-    "mout": {
-        "type": str,
-        "help": "Name of the file where certain matrices will be written",
-        "default": None,
+        "default": "output_file.json",
     },
     "force_partitioning": {"type": float, "help": "Force partitioning", "default": 0.5},
-    "waterEntropy": {"type": bool, "help": "Calculate water entropy", "default": False},
+    "water_entropy": {
+        "type": bool,
+        "help": "Calculate water entropy",
+        "default": False,
+    },
 }
 
 

CodeEntropy/config/data_logger.py

@@ -12,15 +12,15 @@ def __init__(self):
         self.molecule_data = []
         self.residue_data = []
 
-    def save_dataframes_as_json(self, molecule_df, residue_df, outfile):
+    def save_dataframes_as_json(self, molecule_df, residue_df, output_file):
         """Save multiple DataFrames into a single JSON file with separate keys"""
         data = {
             "molecule_data": molecule_df.to_dict(orient="records"),
             "residue_data": residue_df.to_dict(orient="records"),
         }
 
         # Write JSON data to file
-        with open(outfile, "w") as out:
+        with open(output_file, "w") as out:
             json.dump(data, out, indent=4)
 
     def add_results_data(self, molecule, level, type, S_molecule):

CodeEntropy/config/logging_config.py

@@ -4,7 +4,7 @@
 
 
 class LoggingConfig:
-    def __init__(self, folder, default_level=logging.INFO):
+    def __init__(self, folder, log_level=logging.INFO):
         log_directory = os.path.join(folder, "logs")
         os.makedirs(log_directory, exist_ok=True)
 
@@ -26,52 +26,52 @@ def __init__(self, folder, default_level=logging.INFO):
                 "console": {
                     "class": "logging.StreamHandler",
                     "formatter": "simple",
-                    "level": "INFO",
+                    "level": logging.INFO,
                 },
                 "stdout": {
                     "class": "logging.FileHandler",
                     "filename": os.path.join(log_directory, "program.out"),
                     "formatter": "simple",
-                    "level": "INFO",
+                    "level": logging.INFO,
                 },
                 "logfile": {
                     "class": "logging.FileHandler",
                     "filename": os.path.join(log_directory, "program.log"),
                     "formatter": "detailed",
-                    "level": "DEBUG",
+                    "level": logging.DEBUG,
                 },
                 "errorfile": {
                     "class": "logging.FileHandler",
                     "filename": os.path.join(log_directory, "program.err"),
                     "formatter": "detailed",
-                    "level": "ERROR",
+                    "level": logging.ERROR,
                 },
                 "commandfile": {
                     "class": "logging.FileHandler",
                     "filename": os.path.join(log_directory, "program.com"),
                     "formatter": "simple",
-                    "level": "INFO",
+                    "level": logging.INFO,
                 },
                 "mdanalysis_log": {
                     "class": "logging.FileHandler",
                     "filename": os.path.join(log_directory, "mdanalysis.log"),
                     "formatter": "detailed",
-                    "level": "DEBUG",
+                    "level": logging.DEBUG,
                 },
             },
             "loggers": {
                 "": {
                     "handlers": ["console", "stdout", "logfile", "errorfile"],
-                    "level": default_level,
+                    "level": log_level,
                 },
                 "MDAnalysis": {
                     "handlers": ["mdanalysis_log"],
-                    "level": "DEBUG",
+                    "level": logging.DEBUG,
                     "propagate": False,
                 },
                 "commands": {
                     "handlers": ["commandfile"],
-                    "level": "INFO",
+                    "level": logging.INFO,
                     "propagate": False,
                 },
             },
@@ -82,3 +82,23 @@ def setup_logging(self):
         logging.getLogger("MDAnalysis")
         logging.getLogger("commands")
         return logging.getLogger(__name__)
+
+    def update_logging_level(self, log_level):
+        # Update the root logger level
+        root_logger = logging.getLogger()
+        root_logger.setLevel(log_level)
+        for handler in root_logger.handlers:
+            if isinstance(handler, logging.FileHandler):
+                handler.setLevel(logging.DEBUG)
+            else:
+                handler.setLevel(logging.INFO)
+
+        # Update all other loggers and their handlers
+        for logger_name in self.LOGGING["loggers"]:
+            logger = logging.getLogger(logger_name)
+            logger.setLevel(log_level)
+            for handler in logger.handlers:
+                if isinstance(handler, logging.FileHandler):
+                    handler.setLevel(logging.DEBUG)
+                else:
+                    handler.setLevel(logging.INFO)

CodeEntropy/main_mcc.py

@@ -65,26 +65,29 @@ def main():
 
     parser = arg_config.setup_argparse()
     args, unknown = parser.parse_known_args()
-    args.outfile = os.path.join(folder, args.outfile)
-
-    # Determine logging level
-    log_level = logging.DEBUG if args.verbose else logging.INFO
-
-    # Initialize the logging system with the determined log level
-    logging_config = LoggingConfig(folder, default_level=log_level)
-    logger = logging_config.setup_logging()
-
-    # Capture and log the command-line invocation
-    command = " ".join(sys.argv)
-    logging.getLogger("commands").info(command)
+    args.output_file = os.path.join(folder, args.output_file)
 
     try:
+        # Initialize the logging system once
+        logging_config = LoggingConfig(folder)
+        logger = logging_config.setup_logging()
+
         # Process each run in the YAML configuration
         for run_name, run_config in config.items():
             if isinstance(run_config, dict):
                 # Merging CLI arguments with YAML configuration
                 args = arg_config.merge_configs(args, run_config)
 
+                # Determine logging level
+                log_level = logging.DEBUG if args.verbose else logging.INFO
+
+                # Update the logging level
+                logging_config.update_logging_level(log_level)
+
+                # Capture and log the command-line invocation
+                command = " ".join(sys.argv)
+                logging.getLogger("commands").info(command)
+
                 # Ensure necessary arguments are provided
                 if not getattr(args, "top_traj_file"):
                     raise ValueError(
@@ -429,7 +432,7 @@ def main():
             #            'Type':['Orientational (J/mol/K)'],
             #            'Result': [S_orient],})
             #        results_df = pd.concat([results_df, new_row], ignore_index=True)
-            #        with open(args.outfile, "a") as out:
+            #        with open(args.output_file, "a") as out:
             #    print(molecule,
             #          "\t",
             #          level,
@@ -454,7 +457,7 @@ def main():
             molecule, level, "Molecule Total Entropy", S_molecule
         )
         data_logger.save_dataframes_as_json(
-            results_df, residue_results_df, args.outfile
+            results_df, residue_results_df, args.output_file
         )
 
     logger.info("Molecules:")

config.yaml

@@ -7,11 +7,9 @@ run1:
   end: 
   step:
   bin_width:
-  tempra:
+  temperature:
   verbose:
   thread: 
-  outfile:
-  resfile:
-  mout: 
+  output_file: 
   force_partitioning:
-  waterEntropy: 
+  water_entropy:

tests/test_EntropyFunctions/test_arg_config_manager.py

@@ -62,11 +62,9 @@ def setup_file(self, mock_file):
             "tempra: 298.0\n  "
             "verbose: False\n  "
             "thread: 1\n  "
-            "outfile: 'outfile.out'\n  "
-            "resfile: 'res_outfile.out'\n  "
-            "mout: null\n  "
+            "output_file: 'output_file.json'\n  "
             "force_partitioning: 0.5\n  "
-            "waterEntropy: False"
+            "water_entropy: False"
         ).return_value
 
     @patch("builtins.open", new_callable=mock_open)
@@ -126,11 +124,9 @@ def test_invalid_run_config_type(self):
             tempra=298.0,
             verbose=False,
             thread=1,
-            outfile="outfile.out",
-            resfile="res_outfile.out",
-            mout=None,
+            output_file="output_file.json",
             force_partitioning=0.5,
-            waterEntropy=False,
+            water_entropy=False,
         ),
     )
     def test_setup_argparse(self, mock_args):
@@ -271,11 +267,9 @@ def test_merge_configs(self):
             tempra=None,
             verbose=None,
             thread=None,
-            outfile=None,
-            resfile=None,
-            mout=None,
+            output_file=None,
             force_partitioning=None,
-            waterEntropy=None,
+            water_entropy=None,
         )
         run_config = {
             "top_traj_file": ["/path/to/tpr", "/path/to/trr"],
@@ -287,11 +281,9 @@ def test_merge_configs(self):
             "tempra": 298.0,
             "verbose": False,
             "thread": 1,
-            "outfile": "outfile.out",
-            "resfile": "res_outfile.out",
-            "mout": None,
+            "output_file": "output_file.json",
             "force_partitioning": 0.5,
-            "waterEntropy": False,
+            "water_entropy": False,
         }
         merged_args = arg_config.merge_configs(args, run_config)
         self.assertEqual(merged_args.top_traj_file, ["/path/to/tpr", "/path/to/trr"])

tests/test_EntropyFunctions/test_main_mcc.py

@@ -54,11 +54,9 @@ def setup_file(self, mock_file):
             "tempra: 298.0\n  "
             "verbose: False\n  "
             "thread: 1\n  "
-            "outfile: 'outfile.out'\n  "
-            "resfile: 'res_outfile.out'\n  "
-            "mout: null\n  "
+            "output_file: 'output_file.json'\n  "
             "force_partitioning: 0.5\n  "
-            "waterEntropy: False"
+            "water_entropy: False"
         ).return_value
 
     def test_CodeEntropy_imported(self):