test: add "default" semantic to testme.json files

doc/source/testing/testLauncher.rst

@@ -144,7 +144,7 @@ For a single basic configuration one can use :
 .. code-block:: json
 
     {
-        "variants": {
+        "default": {
             "dumpname": "dump.0001.dmp",
             "ini": "idefix.ini",
             "vectPot": false,
@@ -159,7 +159,7 @@ For a single basic configuration one can use :
 Available parameters
 --------------------
 
-The parameters in the ``variants`` dictionnary correspond to the options supported by the
+The parameters in the ``default`` dictionnary correspond to the options supported by the
 :doc:`idfxTest <idfxTest>` script to configure the build and run of *Idefix*.
 
 In addition there is some extra keys which are dedicated to the json interpretation layer :
@@ -195,22 +195,24 @@ In addition there is some extra keys which are dedicated to the json interpretat
 Looping over parameters
 -----------------------
 
-You might want to explore running Idefix within parameter ranges (configuration files, modes).
-For this simply list the values you want as a list. The test script will automatically
-generate all combinations.
+You might want to explore running *Idefix* within parameter ranges (configuration files, modes).
+For this simply list the values you want as a list in tha ``variants`` dictionnary. The test script
+will automatically generate all combinations.
 
 .. code-block:: json
 
     {
-        "variants": {
+        "default": {
             "dumpname": "dump.0001.dmp",
-            "ini": ["idefix.ini","idefix-hll.ini"],
-            "vectPot": [false, true],
             "single": false,
             "reconstruction": 2,
-            "mpi": [false, true],
             "standardTest": false,
             "tolerance": 0
+        },
+        "variants": {
+            "ini": ["idefix.ini","idefix-hll.ini"],
+            "vectPot": [false, true],
+            "mpi": [false, true]
         }
     }
 
@@ -245,25 +247,24 @@ can list several sets as a list. Here using single only on half of the modes.
 .. code-block:: json
 
     {
+        "default": {
+            "dumpname": "dump.0001.dmp",
+            "tolerance": 0
+            "standardTest": false,
+            "reconstruction": 2
+        },
         "variants": [
             {
-                "dumpname": "dump.0001.dmp",
-                "ini": ["idefix.ini"],
+                "ini": "idefix.ini",
                 "vectPot": false,
                 "single": false,
-                "reconstruction": 2,
                 "mpi": [false, true],
-                "standardTest": false,
-                "tolerance": 0
             },{
-                "dumpname": "dump.0001.dmp",
-                "ini": ["idefix-hll.ini"],
+                "ini": "idefix-hll.ini",
                 "vectPot": true,
                 "single": true,
-                "reconstruction": 2,
                 "mpi": [false, true],
-                "standardTest": false,
-                "tolerance": 0
+
             }
         ]
     }
@@ -284,15 +285,17 @@ the order you want to see them composing the test name.
 
     {
         "namings": "ini,single,mpi",
-        "variants": {
+        "default": {
             "dumpname": "dump.0001.dmp",
-            "ini": ["idefix.ini","idefix-hll.ini"],
             "vectPot": false,
-            "single": [false, true],
             "reconstruction": 2,
-            "mpi": [false, true],
             "standardTest": false,
             "tolerance": 0
+        },
+        "variants": {
+            "ini": ["idefix.ini","idefix-hll.ini"],
+            "single": [false, true],
+            "mpi": [false, true],
         }
     }
 
@@ -309,16 +312,18 @@ It is just like if you used and IF statement.
 
     {
         "namings": "ini,single,mpi",
-        "variants": {
+        "default": {
             "dumpname": "dump.0001.dmp",
-            "ini": ["idefix.ini","idefix-hll.ini"],
             "vectPot": false,
-            "single": [false, true],
             "reconstruction": 2,
-            "mpi": [false, true],
             "standardTest": false,
             "tolerance": 0
         },
+        "variants": {
+            "ini": ["idefix.ini","idefix-hll.ini"],
+            "single": [false, true],
+            "mpi": [false, true],
+        },
         "when": {
             "conditions": {
                 "single": true
@@ -348,9 +353,9 @@ They are described like :
 .. code-block:: json
 
     {
-        "variants": {
+        "default": {
             "dumpname": "dump.0001.dmp",
-            "ini": ["idefix.ini"],
+            "ini": "idefix.ini",
             "vectPot": false,
             "single": false,
             "reconstruction": 2,

pytest.ini

@@ -2,5 +2,5 @@
 markers=
   default: Test to run by default.
 python_files="test_*.py"
-junit_logging="system-out"
+junit_logging="all"
 junit_log_passing_tests=false

pytools/idfx_test.py

@@ -169,7 +169,7 @@ def addLog(self, entry):
   def applyConfig(self, config: dict={}):
     # check args
     for key, value in config.items():
-      if key not in ['ini', 'testfile', 'testname', 'dumpname']:
+      if key not in ['ini', 'testfile', 'testname', 'dumpname', 'check_file_produced']:
         assert key in self.cmdArgs, f"The given configuration overriding try to set an invalid paramater : {key}={value}"
 
     # override options

pytools/idfx_test_gen.py

@@ -32,7 +32,7 @@ def __init__(self, currentTestFile: str, name: str = ""):
     self.currentTestName = name
 
   # generate the list of configs to run
-  def genTestConfigs(self, names:str, params, whenClauses = {}) -> list:
+  def genTestConfigs(self, names:str, params, whenClauses = {}, defaultConfig: dict = {}) -> list:
     '''
     Generate the the list of configurations as pytest parameters.
     It will unpack the configuration set by looping on all combinations defined
@@ -48,20 +48,24 @@ def genTestConfigs(self, names:str, params, whenClauses = {}) -> list:
       whenCaluses (dict|list):
         Provide a set of clauses to apply after unpacking
         the configuration so we can patch some values depending on some others.
+      defaultConfig (dict):
+        The default configuration on top of which to apply the variants.
 
     Returns:
       A list of pytest.param() ready to be fiven to parametrized pytest functions.
     '''
     # get name ordering list
     nameList = names.split(',')
+    if '' in nameList:
+      nameList.remove('')
 
     # gen list of complete configs
     all_configs = []
     if isinstance(params, dict):
-      all_configs += self._genOneConfigSeries(names, params)
+      all_configs += self._genOneConfigSeries(names, params, defaultConfig=defaultConfig)
     elif isinstance(params, list):
       for p in params:
-        all_configs += self._genOneConfigSeries(names, p)
+        all_configs += self._genOneConfigSeries(names, p, defaultConfig=defaultConfig)
     else:
       raise Exception("Should never be called !")
 
@@ -155,7 +159,7 @@ def _genNextLevelCombinations(self, input: list, paramName: str, paramValues: li
         result.append(v)
     return result
 
-  def _genOneConfigSeries(self, names: str, config: dict) -> list:
+  def _genOneConfigSeries(self, names: str, config: dict, defaultConfig: dict={}) -> list:
     '''
     Generate the the list of configurations as pytest parameters.
     It will unpack the configuration set by looping on all combinations defined
@@ -167,6 +171,8 @@ def _genOneConfigSeries(self, names: str, config: dict) -> list:
         name of the file.
       config (dict):
         A configuration set as a dictionnary.
+      defaultConfig (dict):
+        The default configuration to overload with the variant part.
 
     Returns:
       A list of pytest.param() ready to be fiven to parametrized pytest functions.
@@ -179,9 +185,11 @@ def _genOneConfigSeries(self, names: str, config: dict) -> list:
     if 'ini' in loopOrder:
       loopOrder.remove('ini')
       loopOrder.append('ini')
+    if '' in loopOrder:
+      loopOrder.remove('')
 
     # init core with everything not a list
-    core = {}
+    core = copy.deepcopy(defaultConfig)
     for key, value in config.items():
       if isinstance(value, list) and key not in DO_NOT_LOOP_ON:
         assert key in nameList, f"All variable parameteres should be ordered in the names list, '{key}' is not."

pytools/idfx_test_run.py

@@ -85,20 +85,27 @@ def genTests(self) -> list:
 
           # load json & build the inner test combinations
           with open(testfilePath, 'r') as fp:
+            # load
             test = json.load(fp)
             idefixTestGenerator=IdefixDirTestGenerator(testfilePath, testfileDir)
+
+            # extract default config & when & variants
+            defaultConfig = test.get('default', {})
+            whenClauses = test.get('when', {})
+            variants = test.get('variants', {})
+
             if 'namings' in test:
               namings = test['namings']
-              autoExtracted = idefixTestGenerator.extractNamingParameters(test['variants'])
+              autoExtracted = idefixTestGenerator.extractNamingParameters(variants)
               self._validateNaming(namings, autoExtracted, testfilePath)
             else:
-              namings = idefixTestGenerator.extractNamingParameters(test['variants'])
+              namings = idefixTestGenerator.extractNamingParameters(variants)
 
             # required to simplify the algos later, if var is listed as variable, we need to loop over it.
-            self._makeVariableArgAsList(namings, test['variants'])
+            self._makeVariableArgAsList(namings, variants)
 
             # gen
-            result += idefixTestGenerator.genTestConfigs(namings, test['variants'], test.get('when', {}))
+            result += idefixTestGenerator.genTestConfigs(namings, variants, whenClauses=whenClauses, defaultConfig=defaultConfig)
         except Exception as e:
           raise Exception(f"Fail to generate tests from {testfileRelPath} : {e}")
 
@@ -151,7 +158,7 @@ def run(self, config: dict) -> None:
 
     # check produced
     for file in check_file_produced:
-      if not os.path.exists(file):
+      if not os.path.exists(file) and not self.currentTestRunner.fake:
         raise Exception(f"Don't find expected file to be produced by the run : {file} !")
 
   def _runNonRegression(self, dumpname, ini, config_override, tolerance=0, definitionFile="", nonReg=True, nonRegIni=None, standardTest=True, first_run_ini=None,first_run_dumpname=None,configure_and_compile=True):

pytools/tests/test/pb1/testme.json

@@ -1,11 +1,13 @@
 {
+  "default": {
+    "dumpname": "dump.0001.dmp",
+    "noplot": true,
+    "reconstruction": 2,
+    "tolerance": 1e-14
+  },
   "variants": [
     {
-      "dumpname": "dump.0001.dmp",
-      "ini": ["idefix.ini","idefix-implicit.ini"],
-      "noplot": true,
-      "reconstruction": 2,
-      "tolerance": 1e-14
+      "ini": ["idefix.ini","idefix-implicit.ini"]
     }
   ]
 }

pytools/tests/test/pb2/testme.json

@@ -1,18 +1,18 @@
 {
+  "default": {
+    "dumpname": "dump.0001.dmp",
+    "reconstruction": 2,
+    "tolerance": 1e-14
+  },
   "variants": [
     {
-      "dumpname": "dump.0001.dmp",
       "ini": ["idefix.ini","idefix-implicit.ini"],
-      "noplot": true,
-      "reconstruction": 2,
-      "tolerance": 1e-14
+      "noplot": true
     },
     {
       "dumpname": "dump.0001.dmp",
       "ini": ["idefix.ini","idefix-implicit.ini"],
-      "noplot": false,
-      "reconstruction": 2,
-      "tolerance": 1e-14
+      "noplot": false
     }
   ]
 }

test/Dust/DustEnergy/testme.json

@@ -1,12 +1,14 @@
 {
   "namings": "ini",
+  "default": {
+    "dumpname": "dump.0001.dmp",
+    "noplot": true,
+    "reconstruction": 2,
+    "tolerance": 0
+  },
   "variants": [
     {
-      "dumpname": "dump.0001.dmp",
-      "ini": ["idefix.ini","idefix-implicit.ini"],
-      "noplot": true,
-      "reconstruction": 2,
-      "tolerance": 0
+      "ini": ["idefix.ini","idefix-implicit.ini"]
     }
   ]
 }

test/Dust/DustyShock/testme.json

@@ -1,11 +1,13 @@
 {
+  "default": {
+    "dumpname": "dump.0001.dmp",
+    "noplot": true,
+    "reconstruction": 2,
+    "tolerance": 1e-14
+  },
   "variants": [
     {
-      "dumpname": "dump.0001.dmp",
-      "ini": ["idefix.ini","idefix-implicit.ini"],
-      "noplot": true,
-      "reconstruction": 2,
-      "tolerance": 1e-14
+      "ini": ["idefix.ini","idefix-implicit.ini"]
     }
   ]
 }

test/Dust/DustyWave/testme.json

@@ -1,12 +1,14 @@
 {
   "namings": "ini",
+  "default": {
+    "dumpname": "dump.0001.dmp",
+    "noplot": true,
+    "reconstruction": 2,
+    "tolerance": 1e-14
+  },
   "variants": [
     {
-      "dumpname": "dump.0001.dmp",
-      "ini": ["idefix.ini","idefix-implicit.ini"],
-      "noplot": true,
-      "reconstruction": 2,
-      "tolerance": 1e-14
+      "ini": ["idefix.ini","idefix-implicit.ini"]
     }
   ]
 }

test/HD/FargoPlanet/testme.json

@@ -1,15 +1,17 @@
 {
   "namings": "ini,mpi",
+  "default": {
+    "dumpname": "dump.0001.dmp",
+    "noplot": true,
+    "reconstruction": 2,
+    "single": false,
+    "dec": [2, 2],
+    "tolerance": 1e-13
+  },
   "variants": [
     {
-      "dumpname": "dump.0001.dmp",
       "ini": ["idefix.ini", "idefix-rkl.ini"],
-      "noplot": true,
-      "reconstruction": 2,
-      "single": false,
-      "mpi": [false, true],
-      "dec": [2, 2],
-      "tolerance": 1e-13
+      "mpi": [false, true]
     }
   ],
   "when": [