Add st2tests.config.db_opts_as_env_vars for itests

Integration tests need to start subprocesses that use the
same database as the test. This matters when running under
pantsbuild, because pants runs several instances of pytest
in parallel. We configured pants to pass an env var to
disambiguate parallel test runs: ST2TESTS_PARALLEL_SLOT.
Then, the db name gets suffixed with this slot number at
runtime. But, when an integration test runs production
code in a subprocess, the production code does not use--
and should not use--any ST2TESTS_* vars, meaning the
subprocess ends up using what is configured in the conf
file instead of the parallel-safe test db.

Thanks to a new-ish oslo_config feature, we can now
update config via env variables. So, make use of that
in integration tests to override the conf-file provided
values with test-provided values.

Author: cognifloyd

st2api/tests/integration/test_gunicorn_configs.py

@@ -22,6 +22,7 @@
 import eventlet
 from eventlet.green import subprocess
 
+import st2tests.config
 from st2common.models.utils import profiling
 from st2common.util.shell import kill_process
 from st2tests.base import IntegrationTestCase
@@ -41,6 +42,7 @@ def test_st2api_wsgi_entry_point(self):
         )
         env = os.environ.copy()
         env["ST2_CONFIG_PATH"] = ST2_CONFIG_PATH
+        env.update(st2tests.config.db_opts_as_env_vars())
         process = subprocess.Popen(cmd, env=env, shell=True, preexec_fn=os.setsid)
         try:
             self.add_process(process=process)
@@ -60,6 +62,7 @@ def test_st2auth(self):
         )
         env = os.environ.copy()
         env["ST2_CONFIG_PATH"] = ST2_CONFIG_PATH
+        env.update(st2tests.config.db_opts_as_env_vars())
         process = subprocess.Popen(cmd, env=env, shell=True, preexec_fn=os.setsid)
         try:
             self.add_process(process=process)

st2common/tests/integration/test_register_content_script.py

@@ -19,6 +19,7 @@
 import sys
 import glob
 
+import st2tests.config
 from st2tests.base import IntegrationTestCase
 from st2common.util.shell import run_command
 from st2tests import config as test_config
@@ -56,7 +57,7 @@ def test_register_from_pack_success(self):
             "--register-runner-dir=%s" % (runner_dirs),
         ]
         cmd = BASE_REGISTER_ACTIONS_CMD_ARGS + opts
-        exit_code, _, stderr = run_command(cmd=cmd)
+        exit_code, _, stderr = self._run_command(cmd=cmd)
         self.assertIn("Registered 3 actions.", stderr)
         self.assertEqual(exit_code, 0)
 
@@ -71,7 +72,7 @@ def test_register_from_pack_fail_on_failure_pack_dir_doesnt_exist(self):
             "--register-no-fail-on-failure",
         ]
         cmd = BASE_REGISTER_ACTIONS_CMD_ARGS + opts
-        exit_code, _, _ = run_command(cmd=cmd)
+        exit_code, _, _ = self._run_command(cmd=cmd)
         self.assertEqual(exit_code, 0)
 
         # Fail on failure, should fail
@@ -81,7 +82,7 @@ def test_register_from_pack_fail_on_failure_pack_dir_doesnt_exist(self):
             "--register-fail-on-failure",
         ]
         cmd = BASE_REGISTER_ACTIONS_CMD_ARGS + opts
-        exit_code, _, stderr = run_command(cmd=cmd)
+        exit_code, _, stderr = self._run_command(cmd=cmd)
         self.assertIn('Directory "doesntexistblah" doesn\'t exist', stderr)
         self.assertEqual(exit_code, 1)
 
@@ -97,7 +98,7 @@ def test_register_from_pack_action_metadata_fails_validation(self):
         ]
 
         cmd = BASE_REGISTER_ACTIONS_CMD_ARGS + opts
-        exit_code, _, stderr = run_command(cmd=cmd)
+        exit_code, _, stderr = self._run_command(cmd=cmd)
         self.assertIn("Registered 0 actions.", stderr)
         self.assertEqual(exit_code, 0)
 
@@ -109,7 +110,7 @@ def test_register_from_pack_action_metadata_fails_validation(self):
             "--register-runner-dir=%s" % (runner_dirs),
         ]
         cmd = BASE_REGISTER_ACTIONS_CMD_ARGS + opts
-        exit_code, _, stderr = run_command(cmd=cmd)
+        exit_code, _, stderr = self._run_command(cmd=cmd)
         self.assertIn("object has no attribute 'get'", stderr)
         self.assertEqual(exit_code, 1)
 
@@ -127,7 +128,7 @@ def test_register_from_packs_doesnt_throw_on_missing_pack_resource_folder(self):
             "-v",
             "--register-sensors",
         ]
-        exit_code, _, stderr = run_command(cmd=cmd)
+        exit_code, _, stderr = self._run_command(cmd=cmd)
         self.assertIn("Registered 0 sensors.", stderr, "Actual stderr: %s" % (stderr))
         self.assertEqual(exit_code, 0)
 
@@ -139,7 +140,7 @@ def test_register_from_packs_doesnt_throw_on_missing_pack_resource_folder(self):
             "--register-all",
             "--register-no-fail-on-failure",
         ]
-        exit_code, _, stderr = run_command(cmd=cmd)
+        exit_code, _, stderr = self._run_command(cmd=cmd)
         self.assertIn("Registered 0 actions.", stderr)
         self.assertIn("Registered 0 sensors.", stderr)
         self.assertIn("Registered 0 rules.", stderr)
@@ -155,7 +156,7 @@ def test_register_all_and_register_setup_virtualenvs(self):
             "--register-setup-virtualenvs",
             "--register-no-fail-on-failure",
         ]
-        exit_code, stdout, stderr = run_command(cmd=cmd)
+        exit_code, stdout, stderr = self._run_command(cmd=cmd)
         self.assertIn("Registering actions", stderr, "Actual stderr: %s" % (stderr))
         self.assertIn("Registering rules", stderr)
         self.assertIn("Setup virtualenv for %s pack(s)" % ("1"), stderr)
@@ -170,7 +171,7 @@ def test_register_setup_virtualenvs(self):
             "--register-setup-virtualenvs",
             "--register-no-fail-on-failure",
         ]
-        exit_code, stdout, stderr = run_command(cmd=cmd)
+        exit_code, stdout, stderr = self._run_command(cmd=cmd)
 
         self.assertIn('Setting up virtualenv for pack "dummy_pack_1"', stderr)
         self.assertIn("Setup virtualenv for 1 pack(s)", stderr)
@@ -186,7 +187,7 @@ def test_register_recreate_virtualenvs(self):
             "--register-setup-virtualenvs",
             "--register-no-fail-on-failure",
         ]
-        exit_code, stdout, stderr = run_command(cmd=cmd)
+        exit_code, stdout, stderr = self._run_command(cmd=cmd)
 
         self.assertIn('Setting up virtualenv for pack "dummy_pack_1"', stderr)
         self.assertIn("Setup virtualenv for 1 pack(s)", stderr)
@@ -200,9 +201,15 @@ def test_register_recreate_virtualenvs(self):
             "--register-recreate-virtualenvs",
             "--register-no-fail-on-failure",
         ]
-        exit_code, stdout, stderr = run_command(cmd=cmd)
+        exit_code, stdout, stderr = self._run_command(cmd=cmd)
 
         self.assertIn('Setting up virtualenv for pack "dummy_pack_1"', stderr)
         self.assertIn("Virtualenv successfully removed.", stderr)
         self.assertIn("Setup virtualenv for 1 pack(s)", stderr)
         self.assertEqual(exit_code, 0)
+
+    @staticmethod
+    def _run_command(cmd):
+        env = os.environ.copy()
+        env.update(st2tests.config.db_opts_as_env_vars())
+        return run_command(cmd=cmd, env=env)

st2common/tests/integration/test_service_setup_log_level_filtering.py

@@ -21,6 +21,7 @@
 import eventlet
 from eventlet.green import subprocess
 
+import st2tests.config
 from st2tests.base import IntegrationTestCase
 from st2tests.fixtures.conf.fixture import FIXTURE_PATH as CONF_FIXTURES_PATH
 
@@ -223,13 +224,16 @@ def test_kombu_heartbeat_tick_log_messages_are_excluded(self):
         stdout = "\n".join(process.stdout.read().decode("utf-8").split("\n"))
         self.assertNotIn("heartbeat_tick", stdout)
 
-    def _start_process(self, config_path, env=None):
+    @staticmethod
+    def _start_process(config_path, env=None):
         cmd = CMD + [config_path]
         cwd = os.path.abspath(os.path.join(BASE_DIR, "../../../"))
         cwd = os.path.abspath(cwd)
+        env = env or os.environ.copy()
+        env.update(st2tests.config.db_opts_as_env_vars())
         process = subprocess.Popen(
             cmd,
-            env=env or os.environ.copy(),
+            env=env,
             cwd=cwd,
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,

st2reactor/tests/integration/test_garbage_collector.py

@@ -20,8 +20,7 @@
 import signal
 import datetime
 
-from oslo_config import cfg
-
+import st2tests.config
 from st2common.util import concurrency
 from st2common.constants import action as action_constants
 from st2common.util import date as date_utils
@@ -277,7 +276,7 @@ def _create_inquiry(self, ttl, timestamp):
     def _start_garbage_collector(self):
         subprocess = concurrency.get_subprocess_module()
         env = os.environ.copy()
-        env["ST2_DATABASE__DB_NAME"] = cfg.CONF.database.db_name
+        env.update(st2tests.config.db_opts_as_env_vars())
         process = subprocess.Popen(
             CMD_INQUIRY,
             stdout=subprocess.PIPE,

st2reactor/tests/integration/test_sensor_container.py

@@ -245,7 +245,7 @@ def test_single_sensor_mode(self):
     def _start_sensor_container(self, cmd=DEFAULT_CMD):
         subprocess = concurrency.get_subprocess_module()
         env = os.environ.copy()
-        env["ST2_DATABASE__DB_NAME"] = cfg.CONF.database.db_name
+        env.update(st2tests.config.db_opts_as_env_vars())
         print("Using command: %s" % (" ".join(cmd)))
         process = subprocess.Popen(
             cmd,

st2tests/st2tests/config.py

@@ -16,6 +16,7 @@
 from __future__ import absolute_import
 
 import os
+from typing import Dict
 
 from oslo_config import cfg, types
 
@@ -92,6 +93,20 @@ def _override_db_opts():
     CONF.set_override(name="host", override="127.0.0.1", group="database")
 
 
+def db_opts_as_env_vars() -> Dict[str, str]:
+    env = {
+        "ST2_DATABASE__HOST": CONF.database.host,
+        "ST2_DATABASE__PORT": str(CONF.database.port),
+        "ST2_DATABASE__DB_NAME": CONF.database.db_name,
+        "ST2_DATABASE__CONNECTION_TIMEOUT": CONF.database.connection_timeout,
+    }
+    if hasattr(CONF.database, "username"):
+        env["ST2_DATABASE__USERNAME"] = CONF.database.username
+    if hasattr(CONF.database, "password"):
+        env["ST2_DATABASE__PASSWORD"] = CONF.database.password
+    return env
+
+
 def _override_common_opts():
     packs_base_path = get_fixtures_packs_base_path()
     CONF.set_override(name="base_path", override=packs_base_path, group="system")