Deps: Update mongoengine to 0.29.1 and pymongo 4.6.3

CHANGELOG.rst

@@ -33,7 +33,7 @@ Fixed
 Changed
 ~~~~~~~
 * Bumped `jsonschema` 2.6.0 -> 3.2.0 now that python3.6 is not supported. #6118
-* Bumped many deps based on the lockfile generated by pants+pex. #6181 #6227 #6200 (by @cognifloyd and @nzlosh)
+* Bumped many deps based on the lockfile generated by pants+pex. #6181 #6227 #6200 #6252 (by @cognifloyd and @nzlosh)
 * Switch to python3's standard lib unittest from unittest2, a backport of python3 unittest features for python2. #6187 (by @nzlosh)
 * Drop Python 3.6 testing in CircleCI. #6080
   Contributed by (@philipphomberger Schwarz IT KG)
@@ -51,6 +51,9 @@ Changed
   and/or `[database].ssl_certfile`, as those options are ignored in StackStorm 3.9.0. #6250
   Contributed by @cognifloyd
 
+* Update mongoengine to 0.29 and pymongo to 4.6.3. The pymongo bump (from 3.x to 4.x) is a major update. #6252
+  Contributed by @cognifloyd
+
 Added
 ~~~~~
 * Continue introducing `pants <https://www.pantsbuild.org/docs>`_ to improve DX (Developer Experience)

Makefile

@@ -60,7 +60,7 @@ ST2TESTS_REDIS_PORT := 6379
 # Pin common pip version here across all the targets
 # Note! Periodic maintenance pip upgrades are required to be up-to-date with the latest pip security fixes and updates
 PIP_VERSION ?= 24.2
-SETUPTOOLS_VERSION ?= 74.1.2
+SETUPTOOLS_VERSION ?= 75.1.0
 PIP_OPTIONS := $(ST2_PIP_OPTIONS)
 
 ifndef PYLINT_CONCURRENCY

fixed-requirements.txt

@@ -7,24 +7,24 @@ cffi==1.17.1
 # NOTE: 2.0 version breaks pymongo work with hosts
 dnspython==1.16.0
 cryptography==43.0.1
-eventlet==0.36.1
+eventlet==0.37.0
 flex==6.14.1
 # Note: installs gitpython==3.1.37 (security fixed) under py3.8 and gitpython==3.1.18 (latest available, vulnerable) under py3.6
 # TODO: Pin to 3.1.37 or higher after dropping python3.6 support
 gitpython==3.1.43
 # Needed by gitpython, old versions used to bundle it
 gitdb==4.0.11
 # Note: greenlet is used by eventlet
-greenlet==3.0.3
+greenlet==3.1.0
 gunicorn==23.0.0
 jsonpath-rw==1.4.0
 jsonschema==3.2.0
-kombu==5.4.0
+kombu==5.4.2
 lockfile==0.12.2
 # Fix MarkupSafe to < 2.1.0 as 2.1.0 removes soft_unicode
 # >=0.23 was from jinja2
 MarkupSafe==2.0.1
-mongoengine==0.23.1
+mongoengine==0.29.1
 # required by orquesta (networkx<2.6 for py3.6, networkx<3 for py3.8)
 networkx==2.8.8
 # networkx dropped its dep on decorator in version 2.6, so the old pin is unneeded.
@@ -34,12 +34,12 @@ decorator==5.1.1
 oslo.config==9.6.0
 oslo.utils==7.3.0
 # paramiko 2.11.0 is needed by cryptography > 37.0.0
-paramiko==3.4.1
+paramiko==3.5.0
 passlib==1.7.4
 # 202403: bump to 3.0.43 for py3.10 support
 prompt-toolkit==3.0.47
 pyinotify==0.9.6 ; platform_system=="Linux"
-pymongo==3.12.3
+pymongo==4.6.3
 pyparsing==3.1.4
 zstandard==0.23.0
 # pyOpenSSL 23.1.0 supports cryptography up to 40.0.x
@@ -49,7 +49,7 @@ editor==1.6.6
 # editor dependency, required here for inclusion in st2client setup.py
 pygments==2.18.0
 python-keyczar==0.716
-pytz==2024.1
+pytz==2024.2
 pywinrm==0.5.0
 pyyaml==6.0.2
 redis==5.0.8
@@ -69,9 +69,9 @@ stevedore==5.3.0
 tenacity==9.0.0
 tooz==6.3.0
 # Note: virtualenv embeds wheels for pip, wheel, and setuptools. So pinning virtualenv pins those as well.
-# virtualenv==20.26.3 (<21) has pip==24.1 wheel==0.43.0 setuptools==70.1.0
-# lockfiles/st2.lock has pip==24.2 wheel==0.43.0 setuptools==72.1.0
-virtualenv==20.26.4
+# virtualenv==20.26.5 (<21) has pip==24.2 wheel==0.44.0 setuptools==75.1.0
+# lockfiles/st2.lock has pip==24.2 wheel==0.44.0 setuptools==75.1.0
+virtualenv==20.26.5
 webob==1.8.8
 zake==0.2.2
 # test requirements below
@@ -84,4 +84,4 @@ psutil==6.0.0
 python-dateutil==2.9.0.post0
 python-statsd==2.1.0
 orjson==3.10.7
-zipp==3.20.1
+zipp==3.20.2

pants-plugins/uses_services/scripts/is_mongo_running.py

@@ -34,12 +34,14 @@ def _is_mongo_running(
         port=db_port,
         connectTimeoutMS=connection_timeout_ms,
         serverSelectionTimeoutMS=connection_timeout_ms,
+        uuidRepresentation="pythonLegacy",
     )
 
     # connection.connect() is lazy. Make a command to test the connection.
     try:
-        # The ismaster command is cheap and does not require auth
-        connection.admin.command("ismaster")
+        # The ping command is cheap and does not require auth
+        # https://www.mongodb.com/community/forums/t/how-to-use-the-new-hello-interface-for-availability/116748/
+        connection.admin.command("ping")
     except (ConnectionFailure, ServerSelectionTimeoutError):
         return False
     return True

requirements-pants.txt

@@ -28,7 +28,8 @@ kombu
 lockfile
 mock
 # mongoengine 0.24.0 has breaking changes to support pymongo 4.0
-mongoengine>=0.21.0,<0.24.0
+# mongoengine 0.29.0 is the first version to officially support mongo 7.0.
+mongoengine>=0.24.0,<0.30.0
 # networkx version is constrained in orquesta.
 networkx
 orjson
@@ -47,7 +48,9 @@ prompt-toolkit
 psutil
 pygments
 # pymongo 3.13 has backports of APIs from pymongo 4 to help w/ migration
-pymongo>=3.11.0,<3.13.0
+# pymongo 4.4 is the first version to officially support mongo 7.0.
+# pymongo 4.7 (or 4.8?) introduces support for standard python logging, which overwhelms our debug logs
+pymongo>=4.0.0,<4.7
 # pyrabbit used in an integration test
 pyrabbit
 pytest

requirements.txt

@@ -18,21 +18,21 @@ cryptography==43.0.1
 decorator==5.1.1
 dnspython==1.16.0
 editor==1.6.6
-eventlet==0.36.1
+eventlet==0.37.0
 flex==6.14.1
 gitdb==4.0.11
 gitpython==3.1.43
-greenlet==3.0.3
+greenlet==3.1.0
 gunicorn==23.0.0
 importlib-metadata==7.1.0
 jinja2==3.1.4
 jsonpath-rw==1.4.0
 jsonschema==3.2.0
-kombu==5.4.0
+kombu==5.4.2
 lockfile==0.12.2
 logshipper@ git+https://github.com/StackStorm/logshipper.git@stackstorm_patched ; platform_system=="Linux"
 mock==5.1.0
-mongoengine==0.23.1
+mongoengine==0.29.1
 networkx==2.8.8
 nose
 nose-parallel==0.4.0
@@ -41,22 +41,22 @@ orjson==3.10.7
 orquesta@ git+https://github.com/StackStorm/orquesta.git@v1.6.0
 oslo.config==9.6.0
 oslo.utils==7.3.0
-paramiko==3.4.1
+paramiko==3.5.0
 passlib==1.7.4
 prettytable==3.10.2
 prompt-toolkit==3.0.47
 psutil==6.0.0
 pyOpenSSL
 pygments==2.18.0
 pyinotify==0.9.6 ; platform_system=="Linux"
-pymongo==3.12.3
+pymongo==4.6.3
 pyparsing==3.1.4
 pyrabbit
 pysocks
 python-dateutil==2.9.0.post0
 python-json-logger
 python-statsd==2.1.0
-pytz==2024.1
+pytz==2024.2
 pywinrm==0.5.0
 pyyaml==6.0.2
 redis==5.0.8
@@ -79,5 +79,5 @@ unittest2
 webob==1.8.8
 webtest
 zake==0.2.2
-zipp==3.20.1
+zipp==3.20.2
 zstandard==0.23.0

st2actions/requirements.txt

@@ -8,10 +8,10 @@
 MarkupSafe==2.0.1
 apscheduler==3.10.4
 chardet==3.0.4
-eventlet==0.36.1
+eventlet==0.37.0
 gitpython==3.1.43
 jinja2==3.1.4
-kombu==5.4.0
+kombu==5.4.2
 lockfile==0.12.2
 logshipper@ git+https://github.com/StackStorm/logshipper.git@stackstorm_patched ; platform_system=="Linux"
 oslo.config==9.6.0

st2api/requirements.txt

@@ -5,14 +5,14 @@
 # If you want to update depdencies for a single component, modify the
 # in-requirements.txt for that component and then run 'make requirements' to
 # update the component requirements.txt
-eventlet==0.36.1
+eventlet==0.37.0
 gunicorn==23.0.0
 jsonschema==3.2.0
-kombu==5.4.0
-mongoengine==0.23.1
+kombu==5.4.2
+mongoengine==0.29.1
 oslo.config==9.6.0
 oslo.utils==7.3.0
-pymongo==3.12.3
+pymongo==4.6.3
 pyparsing==3.1.4
 simplejson
 six==1.16.0

st2auth/requirements.txt

@@ -6,11 +6,11 @@
 # in-requirements.txt for that component and then run 'make requirements' to
 # update the component requirements.txt
 bcrypt==4.2.0
-eventlet==0.36.1
+eventlet==0.37.0
 gunicorn==23.0.0
 oslo.config==9.6.0
 passlib==1.7.4
-pymongo==3.12.3
+pymongo==4.6.3
 six==1.16.0
 st2-auth-backend-flat-file@ git+https://github.com/StackStorm/st2-auth-backend-flat-file.git@master
 st2-auth-ldap@ git+https://github.com/StackStorm/st2-auth-ldap.git@master

st2client/requirements.txt

@@ -20,10 +20,10 @@ pyOpenSSL
 pygments==2.18.0
 pysocks
 python-dateutil==2.9.0.post0
-pytz==2024.1
+pytz==2024.2
 pyyaml==6.0.2
 requests==2.32.3
 six==1.16.0
 sseclient-py==1.8.0
 typing-extensions==4.12.2
-zipp==3.20.1
+zipp==3.20.2

st2common/bin/migrations/v3.8/st2-drop-st2exporter-marker-collections

@@ -37,7 +37,7 @@ DUMPER_MARKER_COLLECTION = "dumper_marker_d_b"
 
 def delete_marker_collections():
     db = get_db()
-    collections = db.collection_names()
+    collections = db.list_collection_names()
 
     if MARKER_COLLECTION in collections:
         print(f"Dropping {MARKER_COLLECTION} collection...")

st2common/requirements.txt

@@ -14,24 +14,24 @@ ciso8601
 cryptography==43.0.1
 decorator==5.1.1
 dnspython==1.16.0
-eventlet==0.36.1
+eventlet==0.37.0
 flex==6.14.1
 gitdb==4.0.11
 gitpython==3.1.43
-greenlet==3.0.3
+greenlet==3.1.0
 jinja2==3.1.4
 jsonpath-rw==1.4.0
 jsonschema==3.2.0
-kombu==5.4.0
+kombu==5.4.2
 lockfile==0.12.2
-mongoengine==0.23.1
+mongoengine==0.29.1
 networkx==2.8.8
 orjson==3.10.7
 orquesta@ git+https://github.com/StackStorm/orquesta.git@v1.6.0
 oslo.config==9.6.0
-paramiko==3.4.1
+paramiko==3.5.0
 pyOpenSSL
-pymongo==3.12.3
+pymongo==4.6.3
 python-dateutil==2.9.0.post0
 python-statsd==2.1.0
 pyyaml==6.0.2

st2common/st2common/models/db/__init__.py

@@ -188,7 +188,16 @@ def _db_connect(
     # 30 seconds, which means it will block up to 30 seconds and fail if there are any SSL related
     # or other errors
     connection_timeout = cfg.CONF.database.connection_timeout
+
+    # TODO: Add uuid_representation option in st2.conf + a migration guide/script.
+    # This preserves the uuid handling from pymongo 3.x, but it is not portable:
+    # https://pymongo.readthedocs.io/en/stable/examples/uuid.html#handling-uuid-data-example
+    uuid_representation = "pythonLegacy"
+
     connection = mongoengine.connection.connect(
+        # kwargs are defined by mongoengine and pymongo.MongoClient:
+        # https://docs.mongoengine.org/apireference.html#mongoengine.connect
+        # https://pymongo.readthedocs.io/en/stable/api/pymongo/mongo_client.html#pymongo.mongo_client.MongoClient
         db_name,
         host=db_host,
         port=db_port,
@@ -197,6 +206,7 @@ def _db_connect(
         password=password,
         connectTimeoutMS=connection_timeout,
         serverSelectionTimeoutMS=connection_timeout,
+        uuidRepresentation=uuid_representation,
         **tls_kwargs,
         **compressor_kwargs,
     )

st2common/st2common/models/db/stormbase.py

@@ -20,6 +20,7 @@
 import bson
 import six
 import mongoengine as me
+from mongoengine.pymongo_support import LEGACY_JSON_OPTIONS
 from oslo_config import cfg
 
 from st2common.util import mongoescape
@@ -105,7 +106,8 @@ def to_serializable_dict(self, mask_secrets=False):
             if isinstance(v, JSON_UNFRIENDLY_TYPES):
                 v = str(v)
             elif isinstance(v, me.EmbeddedDocument):
-                v = json_decode(v.to_json())
+                # TODO: Allow overriding json_options.uuid_representation via cfg
+                v = json_decode(v.to_json(json_options=LEGACY_JSON_OPTIONS))
 
             serializable_dict[k] = v
 

st2common/tests/unit/base.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 
 from __future__ import absolute_import
+
 import time
 
 import mongoengine

st2common/tests/unit/test_db.py

@@ -109,6 +109,13 @@ def tearDown(self):
         disconnect()
         cfg.CONF.reset()
 
+    @classmethod
+    def tearDownClass(cls):
+        # since tearDown discconnects, dropping the database in tearDownClass
+        # fails withotu establishing a new connection.
+        cls._establish_connection_and_re_create_db()
+        super().tearDownClass()
+
     def test_check_connect(self):
         """
         Tests connectivity to the db server. Requires the db server to be
@@ -378,6 +385,7 @@ def test_db_setup(self, mock_mongoengine):
                 "tlsAllowInvalidHostnames": False,
                 "connectTimeoutMS": 3000,
                 "serverSelectionTimeoutMS": 3000,
+                "uuidRepresentation": "pythonLegacy",
             },
         )
 
@@ -615,11 +623,13 @@ def test_cleanup(self):
         """
         Tests dropping the database. Requires the db server to be running.
         """
-        self.assertIn(cfg.CONF.database.db_name, self.db_connection.database_names())
+        self.assertIn(
+            cfg.CONF.database.db_name, self.db_connection.list_database_names()
+        )
 
         connection = db_cleanup()
 
-        self.assertNotIn(cfg.CONF.database.db_name, connection.database_names())
+        self.assertNotIn(cfg.CONF.database.db_name, connection.list_database_names())
 
 
 @mock.patch.object(PoolPublisher, "publish", mock.MagicMock())

st2reactor/requirements.txt

@@ -6,10 +6,10 @@
 # in-requirements.txt for that component and then run 'make requirements' to
 # update the component requirements.txt
 apscheduler==3.10.4
-eventlet==0.36.1
+eventlet==0.37.0
 jsonpath-rw==1.4.0
 jsonschema==3.2.0
-kombu==5.4.0
+kombu==5.4.2
 oslo.config==9.6.0
 python-dateutil==2.9.0.post0
 six==1.16.0