Add MySQL metastore backend

.env.example

@@ -3,8 +3,14 @@ POSTGRES_USER=quickwit-dev
 POSTGRES_PASSWORD=quickwit-dev
 POSTGRES_DB=quickwit-metastore-dev
 
+# Change the user name and password of the mysql database
+MYSQL_USER=quickwit-dev
+MYSQL_PASSWORD=quickwit-dev
+MYSQL_DB=quickwit-metastore-dev
+
 # Update all services to the latest versions
 LOCALSTACK_VERSION=latest
+MYSQL_VERSION=latest
 POSTGRES_VERSION=latest
 PULSAR_VERSION=latest
 CP_VERSION=latest
@@ -23,4 +29,5 @@ MAP_HOST_AZURITE=0.0.0.0
 MAP_HOST_GRAFANA=0.0.0.0
 MAP_HOST_JAEGER=0.0.0.0
 MAP_HOST_OTEL=0.0.0.0
+MAP_HOST_MYSQL=0.0.0.0
 MAP_HOST_PROMETHEUS=0.0.0.0

Makefile

@@ -38,8 +38,11 @@ docker-compose-monitoring:
 docker-rm-postgres-volume:
 	docker volume rm quickwit_postgres_data
 
+docker-rm-mysql-volume:
+	docker volume rm quickwit_mysql_data
+
 docker-rm-volumes:
-	docker volume rm quickwit_azurite_data quickwit_fake_gcs_server_data quickwit_grafana_conf quickwit_grafana_data quickwit_localstack_data quickwit_postgres_data
+	docker volume rm quickwit_azurite_data quickwit_fake_gcs_server_data quickwit_grafana_conf quickwit_grafana_data quickwit_localstack_data quickwit_mysql_data quickwit_postgres_data
 
 doc:
 	@$(MAKE) -C $(QUICKWIT_SRC) doc

docker-compose.yml

@@ -69,6 +69,33 @@ services:
       timeout: 5s
       retries: 100
 
+  mysql:
+    # MySQL 8.0 is the minimum supported version.
+    image: mysql:${MYSQL_VERSION:-8.0}
+    container_name: mysql
+    ports:
+      - "${MAP_HOST_MYSQL:-127.0.0.1}:3307:3306"
+    profiles:
+      - all
+      - mysql
+    environment:
+      MYSQL_DATABASE: ${MYSQL_DB:-quickwit-metastore-dev}
+      MYSQL_USER: ${MYSQL_USER:-quickwit-dev}
+      MYSQL_PASSWORD: ${MYSQL_PASSWORD:-quickwit-dev}
+      MYSQL_ROOT_PASSWORD: root
+      TZ: UTC
+    command: >
+      --default-authentication-plugin=mysql_native_password
+      --default-time-zone=+00:00
+      --sql-mode=STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION
+    volumes:
+      - mysql_data:/var/lib/mysql
+    healthcheck:
+      test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "quickwit-dev", "-pquickwit-dev"]
+      interval: 1s
+      timeout: 5s
+      retries: 100
+
   pulsar-broker:
     # The oldest version with arm64 docker images. EOL May 2 2025
     image: apachepulsar/pulsar:${PULSAR_VERSION:-3.0.0}
@@ -230,4 +257,5 @@ volumes:
   grafana_conf:
   grafana_data:
   localstack_data:
+  mysql_data:
   postgres_data:

quickwit/Cargo.toml

@@ -217,6 +217,7 @@ rustls-pemfile = "2.2"
 sea-query = { version = "0.32" }
 sea-query-binder = { version = "0.7", features = [
   "runtime-tokio-rustls",
+  "sqlx-mysql",
   "sqlx-postgres",
 ] }
 # ^1.0.184 due to serde-rs/serde#2538
@@ -232,6 +233,7 @@ siphasher = "1.0"
 smallvec = "1"
 sqlx = { version = "0.8", features = [
   "migrate",
+  "mysql",
   "postgres",
   "runtime-tokio-rustls",
   "time",
@@ -290,6 +292,7 @@ tracing-subscriber = { version = "0.3", features = [
 ttl_cache = "0.5"
 typetag = "0.2"
 ulid = "1.2"
+url = "2.5"
 ureq = "3"
 username = "0.2"
 # We cannot upgrade to utoipa 5.0+ due to significant breaking changes:
@@ -308,6 +311,7 @@ wiremock = "0.6"
 zstd = { version = "0.13", default-features = false }
 
 aws-config = "1.8"
+aws-sigv4 = "1.3"
 aws-credential-types = { version = "1.2", features = ["hardcoded-credentials"] }
 aws-runtime = "1.5"
 aws-sdk-kinesis = "1.97"

quickwit/Makefile

@@ -39,6 +39,7 @@ test-all:
 	QW_S3_ENDPOINT=http://localhost:4566 \
 	QW_S3_FORCE_PATH_STYLE_ACCESS=1 \
 	QW_TEST_DATABASE_URL=postgres://quickwit-dev:quickwit-dev@localhost:5432/quickwit-metastore-dev \
+	QW_TEST_MYSQL_DATABASE_URL=mysql://quickwit-dev:quickwit-dev@localhost:3307/quickwit-metastore-dev \
 	RUST_MIN_STACK=67108864 \
 	cargo nextest run --all-features --retries 5
 	cargo nextest run --test failpoints --features fail/failpoints

quickwit/quickwit-cli/Cargo.toml

@@ -105,6 +105,7 @@ release-feature-set = [
   "quickwit-serve/lambda",
   "quickwit-storage/azure",
   "quickwit-storage/gcs",
+  "quickwit-metastore/mysql",
   "quickwit-metastore/postgres",
 ]
 release-feature-vendored-set = [
@@ -119,6 +120,7 @@ release-feature-vendored-set = [
   "quickwit-serve/lambda",
   "quickwit-storage/azure",
   "quickwit-storage/gcs",
+  "quickwit-metastore/mysql",
   "quickwit-metastore/postgres",
 ]
 release-macos-feature-vendored-set = [
@@ -132,6 +134,7 @@ release-macos-feature-vendored-set = [
   "quickwit-serve/lambda",
   "quickwit-storage/azure",
   "quickwit-storage/gcs",
+  "quickwit-metastore/mysql",
   "quickwit-metastore/postgres",
 ]
 release-jemalloc-profiled = [

quickwit/quickwit-common/src/uri.rs

@@ -37,6 +37,7 @@ pub enum Protocol {
     Ram = 6,
     S3 = 7,
     Google = 8,
+    MySQL = 9,
 }
 
 impl Protocol {
@@ -50,6 +51,7 @@ impl Protocol {
             Protocol::Ram => "ram",
             Protocol::S3 => "s3",
             Protocol::Google => "gs",
+            Protocol::MySQL => "mysql",
         }
     }
 
@@ -66,7 +68,7 @@ impl Protocol {
     }
 
     pub fn is_database(&self) -> bool {
-        matches!(&self, Protocol::PostgreSQL)
+        matches!(&self, Protocol::PostgreSQL | Protocol::MySQL)
     }
 }
 
@@ -86,6 +88,7 @@ impl FromStr for Protocol {
             "grpc" => Ok(Protocol::Grpc),
             "actor" => Ok(Protocol::Actor),
             "pg" | "postgres" | "postgresql" => Ok(Protocol::PostgreSQL),
+            "mysql" => Ok(Protocol::MySQL),
             "ram" => Ok(Protocol::Ram),
             "s3" => Ok(Protocol::S3),
             "gs" => Ok(Protocol::Google),
@@ -199,7 +202,7 @@ impl Uri {
 
     /// Returns the last component of the URI.
     pub fn file_name(&self) -> Option<&Path> {
-        if self.protocol() == Protocol::PostgreSQL {
+        if self.protocol().is_database() {
             return None;
         }
         let path = self.path();
@@ -236,8 +239,8 @@ impl Uri {
                 .join(path)
                 .to_string_lossy()
                 .to_string(),
-            Protocol::PostgreSQL => bail!(
-                "cannot join PostgreSQL URI `{}` with path `{:?}`",
+            Protocol::PostgreSQL | Protocol::MySQL => bail!(
+                "cannot join database URI `{}` with path `{:?}`",
                 self.uri,
                 path
             ),
@@ -573,6 +576,9 @@ mod tests {
         Uri::for_test("postgres://username:password@localhost:5432/metastore")
             .join("table")
             .unwrap_err();
+        Uri::for_test("mysql://username:password@localhost:3306/metastore")
+            .join("table")
+            .unwrap_err();
     }
 
     #[test]
@@ -589,6 +595,11 @@ mod tests {
                 .parent()
                 .is_none()
         );
+        assert!(
+            Uri::for_test("mysql://localhost:3306/db")
+                .parent()
+                .is_none()
+        );
 
         assert!(Uri::for_test("ram:///").parent().is_none());
         assert_eq!(Uri::for_test("ram:///foo").parent().unwrap(), "ram:///");
@@ -681,6 +692,11 @@ mod tests {
                 .file_name()
                 .is_none()
         );
+        assert!(
+            Uri::for_test("mysql://localhost:3306/db")
+                .file_name()
+                .is_none()
+        );
 
         assert!(Uri::for_test("ram:///").file_name().is_none());
         assert_eq!(
@@ -804,6 +820,17 @@ mod tests {
                 );
             }
         }
+        assert_eq!(
+            Uri::for_test("mysql://localhost:3306/metastore").as_redacted_str(),
+            "mysql://localhost:3306/metastore"
+        );
+        {
+            let uri = Uri::from_str("mysql://username:password@localhost:3306/metastore").unwrap();
+            let expected_uri =
+                "mysql://username:***redacted***@localhost:3306/metastore".to_string();
+            assert_eq!(uri.as_redacted_str(), expected_uri);
+            assert_eq!(format!("{uri}"), expected_uri);
+        }
     }
 
     #[test]

quickwit/quickwit-config/src/lib.rs

@@ -70,7 +70,8 @@ use crate::merge_policy_config::{
     ConstWriteAmplificationMergePolicyConfig, MergePolicyConfig, StableLogMergePolicyConfig,
 };
 pub use crate::metastore_config::{
-    MetastoreBackend, MetastoreConfig, MetastoreConfigs, PostgresMetastoreConfig,
+    MetastoreBackend, MetastoreConfig, MetastoreConfigs, MysqlAuthMode, MysqlMetastoreConfig,
+    PostgresMetastoreConfig,
 };
 pub use crate::node_config::{
     CacheConfig, CachePolicy, DEFAULT_QW_CONFIG_PATH, GrpcConfig, IndexerConfig, IngestApiConfig,