diff --git a/_includes/async-replication-per-collection-config.mdx b/_includes/async-replication-per-collection-config.mdx index 14cb1dd7f..2e8f141cf 100644 --- a/_includes/async-replication-per-collection-config.mdx +++ b/_includes/async-replication-per-collection-config.mdx @@ -1,3 +1,3 @@ :::info Collection-level configuration — Added in `v1.36` -Async replication parameters can also be set per-collection via the `asyncConfig` object in `replicationConfig`. Per-collection settings override the cluster-wide environment variable defaults. See [Collection `asyncConfig` parameters](/weaviate/config-refs/collections#async-config) for details. +Async replication runs by default for any collection with a replication factor greater than `1` (as of `v1.38`). To fine-tune its behavior for a specific collection, set the `asyncConfig` object in `replicationConfig`. Cluster-wide environment variable settings override per-collection settings. See [Collection `asyncConfig` parameters](/weaviate/config-refs/collections#async-config) for details. ::: diff --git a/_includes/code/config-refs/reference.collections.py b/_includes/code/config-refs/reference.collections.py index 80d29c11c..5f1871bd9 100644 --- a/_includes/code/config-refs/reference.collections.py +++ b/_includes/code/config-refs/reference.collections.py @@ -57,7 +57,6 @@ ), replication_config=Configure.replication( factor=1, - async_enabled=False, deletion_strategy=ReplicationDeletionStrategy.TIME_BASED_RESOLUTION, ), ) @@ -733,7 +732,6 @@ # highlight-start replication_config=Configure.replication( factor=3, - async_enabled=True, deletion_strategy=ReplicationDeletionStrategy.TIME_BASED_RESOLUTION, ), # highlight-end @@ -743,7 +741,6 @@ # Test collection = client.collections.use("Article") config = collection.config.get() -assert config.replication_config.async_enabled == True assert ( config.replication_config.deletion_strategy == ReplicationDeletionStrategy.TIME_BASED_RESOLUTION diff --git a/_includes/code/configuration/replication-consistency.mdx b/_includes/code/configuration/replication-consistency.mdx index 9e1e6d0cc..1392a7afd 100644 --- a/_includes/code/configuration/replication-consistency.mdx +++ b/_includes/code/configuration/replication-consistency.mdx @@ -43,8 +43,7 @@ curl \ } ], "replicationConfig": { - "factor": 3, - "asyncEnabled": true + "factor": 3 } }' \ http://localhost:8080/v1/schema diff --git a/_includes/code/howto/go/docs/manage-data.classes_test.go b/_includes/code/howto/go/docs/manage-data.classes_test.go index a6f07668b..cfc4209c2 100644 --- a/_includes/code/howto/go/docs/manage-data.classes_test.go +++ b/_includes/code/howto/go/docs/manage-data.classes_test.go @@ -585,8 +585,8 @@ func Test_ManageDataClasses(t *testing.T) { articleClass := &models.Class{ Class: "Article", Description: "Collection of articles", + // Async replication runs by default when the replication factor is greater than 1 ReplicationConfig: &models.ReplicationConfig{ - AsyncEnabled: true, Factor: 3, DeletionStrategy: models.ReplicationConfigDeletionStrategyTimeBasedResolution, }, diff --git a/_includes/code/howto/manage-data.collections.py b/_includes/code/howto/manage-data.collections.py index ffab3b269..994eac792 100644 --- a/_includes/code/howto/manage-data.collections.py +++ b/_includes/code/howto/manage-data.collections.py @@ -786,9 +786,9 @@ client.collections.create( "Article", # highlight-start + # Async replication runs by default when the replication factor is greater than 1 replication_config=Configure.replication( factor=3, - async_enabled=True, ), # highlight-end ) @@ -819,10 +819,8 @@ # highlight-start replication_config=Configure.replication( factor=3, - async_enabled=True, deletion_strategy=ReplicationDeletionStrategy.TIME_BASED_RESOLUTION, async_config=Configure.Replication.async_config( - max_workers=5, hashtree_height=16, frequency=30, ), @@ -840,7 +838,6 @@ collection.config.update( replication_config=Reconfigure.replication( async_config=Reconfigure.Replication.async_config( - max_workers=10, frequency=60, ), ), @@ -851,7 +848,6 @@ # Test collection = client.collections.use("Article") config = collection.config.get() -assert config.replication_config.async_enabled == True assert ( config.replication_config.deletion_strategy == ReplicationDeletionStrategy.TIME_BASED_RESOLUTION diff --git a/_includes/code/howto/manage-data.collections.ts b/_includes/code/howto/manage-data.collections.ts index b12ce8e1c..8418cf75a 100644 --- a/_includes/code/howto/manage-data.collections.ts +++ b/_includes/code/howto/manage-data.collections.ts @@ -677,9 +677,9 @@ import { configure } from 'weaviate-client'; await client.collections.create({ name: 'Article', // highlight-start + // Async replication runs by default when the replication factor is greater than 1 replication: configure.replication({ factor: 1, - asyncEnabled: true, }), // highlight-end }) @@ -710,10 +710,8 @@ await replicationClient.collections.create({ // highlight-start replication: configure.replication({ factor: 3, - asyncEnabled: true, deletionStrategy: 'TimeBasedResolution', asyncConfig: { - maxWorkers: 5, hashtreeHeight: 16, frequency: 30, }, @@ -729,7 +727,6 @@ const articleReplication = replicationClient.collections.use('Article') await articleReplication.config.update({ replication: reconfigure.replication({ asyncConfig: { - maxWorkers: 10, frequency: 60, }, }), diff --git a/_includes/code/howto/search.boost.py b/_includes/code/howto/search.boost.py new file mode 100644 index 000000000..cb91f51d9 --- /dev/null +++ b/_includes/code/howto/search.boost.py @@ -0,0 +1,240 @@ +# How-to: Search > Boost results — Python examples. +# +# Requires Weaviate v1.38+ and the Python client release that adds Boost +# support (PR weaviate/weaviate-python-client#2030). Boost is gRPC-only — +# REST/curl is not supported. +# +# Uses the text2vec-transformers vectorizer. Run against the local stack +# in tests/docker-compose-anon.yml (Weaviate + transformers inference). + +import time +from datetime import datetime, timedelta, timezone + +import weaviate +from weaviate.classes.config import Configure, DataType, Property, Tokenization +from weaviate.classes.query import Boost, Filter + +client = weaviate.connect_to_local() + +# ---- Fixture: an Articles collection with date + numeric properties ---- +client.collections.delete("Articles") +client.collections.create( + name="Articles", + vector_config=Configure.Vectors.text2vec_transformers(), + properties=[ + Property(name="title", data_type=DataType.TEXT), + Property(name="category", data_type=DataType.TEXT, tokenization=Tokenization.FIELD), + Property(name="published", data_type=DataType.DATE), + Property(name="likes", data_type=DataType.INT), + Property(name="price", data_type=DataType.NUMBER), + Property(name="draft", data_type=DataType.BOOL), + ], +) + +now = datetime.now(timezone.utc) +articles = client.collections.use("Articles") +articles.data.insert_many([ + {"title": "Transformers explained", "category": "research", "published": now - timedelta(days=2), "likes": 100, "price": 49.99, "draft": False}, + {"title": "Old transformer survey", "category": "research", "published": now - timedelta(days=400), "likes": 5000, "price": 49.99, "draft": False}, + {"title": "How to fine-tune a model", "category": "tutorial", "published": now - timedelta(days=1), "likes": 30, "price": 9.99, "draft": False}, + {"title": "Pricing transformers", "category": "tutorial", "published": now - timedelta(days=10), "likes": 5000000, "price": 199.0, "draft": False}, + {"title": "Draft: transformer architecture","category": "research", "published": now - timedelta(days=3), "likes": 200, "price": 9.99, "draft": True}, +]) + +# Wait briefly for the vectorizer to finish indexing the new objects. +time.sleep(3) + + +# ========================================== +# ===== Filter boost (soft WHERE) ===== +# ========================================== + +# START BoostFilter +# Promote articles in the "research" category without filtering others out. +response = articles.query.near_text( + query="transformer architectures", + limit=5, + # highlight-start + boost=Boost.filter( + Filter.by_property("category").equal("research"), + weight=0.5, + ), + # highlight-end + return_properties=["title", "category"], +) + +for o in response.objects: + print(o.properties["category"], "-", o.properties["title"]) +# END BoostFilter + +assert response.objects[0].properties["category"] == "research" + + +# ========================================== +# ===== Property boost (numeric value) ===== +# ========================================== + +# START BoostProperty +# Bias toward articles with more `likes`. LOG1P dampens the long tail so a +# single 5-million-likes outlier doesn't dominate. +response = articles.query.near_text( + query="transformer architectures", + limit=5, + # highlight-start + boost=Boost.property( + "likes", + modifier=Boost.Modifier.LOG1P, + weight=0.7, + ), + # highlight-end + return_properties=["title", "likes"], +) + +for o in response.objects: + print(o.properties["likes"], "-", o.properties["title"]) +# END BoostProperty + + +# ========================================== +# ===== Time decay (boost recent docs) ===== +# ========================================== + +# START BoostTimeDecay +# Score decays exponentially over time. "30d scale" + decay=0.5 means an +# article that's 30 days old gets half the score of one published "now". +response = articles.query.near_text( + query="transformer architectures", + limit=5, + # highlight-start + boost=Boost.time_decay( + "published", + origin="now", + scale=timedelta(days=30), + curve=Boost.Curve.EXPONENTIAL, + decay=0.5, + weight=0.6, + ), + # highlight-end + return_properties=["title", "published"], +) +# END BoostTimeDecay + +# The 400-day-old "Old transformer survey" should be demoted vs the 2-day-old article. +top_titles = [o.properties["title"] for o in response.objects[:2]] +assert "Old transformer survey" not in top_titles + + +# ========================================== +# ===== Numeric decay (closest to a value) ===== +# ========================================== + +# START BoostNumericDecay +# Score peaks at a target price and falls off symmetrically. Gauss gives a +# bell-shaped falloff: items within `offset` of $49.99 score 1.0, items at +# $59.99 (one scale away) score `decay`. +response = articles.query.near_text( + query="transformer architectures", + limit=5, + # highlight-start + boost=Boost.numeric_decay( + "price", + origin=49.99, + scale=10.0, + curve=Boost.Curve.GAUSSIAN, + decay=0.5, + weight=0.5, + ), + # highlight-end + return_properties=["title", "price"], +) +# END BoostNumericDecay + +# Both top-2 results have the target price; the $199 outlier is pushed down. +assert all(o.properties["price"] == 49.99 for o in response.objects[:2]) + + +# ========================================== +# ===== Blend multiple conditions ===== +# ========================================== + +# START BoostBlend +# Combine two soft signals: recency (weight 2) + popularity (weight 1). +# The outer weight=0.4 controls how much the blended rank affects the +# final score; the inner weights are *per-condition* and balance each +# other. +response = articles.query.near_text( + query="transformer architectures", + limit=5, + # highlight-start + boost=Boost.blend( + Boost.time_decay("published", origin="now", scale=timedelta(days=30), weight=2.0), + Boost.property("likes", modifier=Boost.Modifier.LOG1P, weight=1.0), + weight=0.4, + depth=200, # rescore the top 200 vector matches + ), + # highlight-end + return_properties=["title", "likes", "published"], +) +# END BoostBlend + +# Recency dominates, but the LOG1P-dampened 5M-likes article still surfaces +# in the top 3 — popularity is helping, not invisible. +assert any(o.properties["likes"] == 5_000_000 for o in response.objects[:3]) + + +# ========================================== +# ===== Negative weights demote ===== +# ========================================== + +# START BoostNegativeWeight +# A negative per-condition weight pushes matching documents DOWN — they +# stay in the result set but lose ground against everything else. Use +# this to deprioritize drafts without filtering them out entirely. +response = articles.query.bm25( + query="transformer", + limit=5, + # highlight-start + boost=Boost.blend( + Boost.filter(Filter.by_property("draft").equal(True), weight=-2.0), + weight=0.5, + ), + # highlight-end + return_properties=["title", "draft"], +) + +# The draft article is still in results, just no longer first. +all_titles = [o.properties["title"] for o in response.objects] +assert any("Draft" in t for t in all_titles) +assert response.objects[0].properties["draft"] is False +# END BoostNegativeWeight + + +# ========================================== +# ===== Boost on hybrid search ===== +# ========================================== + +# START BoostOnHybrid +# Hybrid keeps its own alpha-blend of BM25 + vector. The boost runs once +# over the fused hybrid result — the sub-search legs don't see it. +response = articles.query.hybrid( + query="transformer architectures", + alpha=0.75, + limit=5, + # highlight-start + boost=Boost.blend( + Boost.filter(Filter.by_property("category").equal("research"), weight=1.0), + Boost.filter(Filter.by_property("draft").equal(True), weight=-2.0), + weight=0.3, + ), + # highlight-end + return_properties=["title", "category", "draft"], +) +# END BoostOnHybrid + +# Top result is research and not a draft — both boost legs worked. +assert response.objects[0].properties["category"] == "research" +assert response.objects[0].properties["draft"] is False + + +client.collections.delete("Articles") +client.close() diff --git a/_includes/code/howto/search.filters.nested.py b/_includes/code/howto/search.filters.nested.py new file mode 100644 index 000000000..1db3c2a22 --- /dev/null +++ b/_includes/code/howto/search.filters.nested.py @@ -0,0 +1,162 @@ +# Howto: Search -> Filters on nested object properties - Python examples. +# +# Preview feature: requires Weaviate v1.38+ with +# `WEAVIATE_PREVIEW_NESTED_FILTERING=on` set on the server. Released +# Weaviate versions reject `cars.make`-style nested paths at the filter +# parser. Not wired into pytest CI yet — promote at GA. + +import weaviate +from weaviate.classes.config import Configure, Property, DataType, Tokenization +from weaviate.classes.query import Filter + +client = weaviate.connect_to_local() + +client.collections.delete("Document") + +# Schema: Document.cars (object[]) -> tires (object[]). +# Mirrors the path patterns used by the worked examples below +# (cars.make, cars[0].make, cars.tires.width, ...). +client.collections.create( + name="Document", + vector_config=Configure.Vectors.self_provided(), + inverted_index_config=Configure.inverted_index(index_null_state=True), + properties=[ + Property(name="title", data_type=DataType.TEXT, tokenization=Tokenization.FIELD), + Property( + name="cars", + data_type=DataType.OBJECT_ARRAY, + nested_properties=[ + Property(name="make", data_type=DataType.TEXT, tokenization=Tokenization.FIELD), + Property(name="color", data_type=DataType.TEXT, tokenization=Tokenization.FIELD), + Property( + name="tires", + data_type=DataType.OBJECT_ARRAY, + nested_properties=[ + Property(name="brand", data_type=DataType.TEXT, tokenization=Tokenization.FIELD), + Property(name="width", data_type=DataType.INT), + ], + ), + ], + ), + ], +) + +docs = client.collections.use("Document") +docs.data.insert_many([ + # Doc 1: two cars; (Toyota, red) + (Honda, blue) + {"title": "doc1", "cars": [ + {"make": "Toyota", "color": "red", + "tires": [{"brand": "Bridgestone", "width": 215}, + {"brand": "Bridgestone", "width": 215}]}, + {"make": "Honda", "color": "blue", + "tires": [{"brand": "Pirelli", "width": 205}, + {"brand": "Pirelli", "width": 205}]}, + ]}, + # Doc 2: one Toyota, no tires + {"title": "doc2", "cars": [ + {"make": "Toyota", "color": "blue"}, + ]}, + # Doc 3: one Honda (red) with wide Michelin tires + {"title": "doc3", "cars": [ + {"make": "Honda", "color": "red", + "tires": [{"brand": "Michelin", "width": 250}, + {"brand": "Michelin", "width": 250}]}, + ]}, +]) + + +# ========================================== +# ===== Existential match (any element) ===== +# ========================================== + +# START NestedExistential +# "any car has make = Toyota" — matches Doc 1 (first car) and Doc 2 (only car) +response = docs.query.fetch_objects( + # highlight-start + filters=Filter.by_property("cars.make").equal("Toyota"), + # highlight-end + return_properties=["title"], +) + +for o in response.objects: + print(o.properties) +# END NestedExistential + +assert len(response.objects) == 2 + + +# ========================================== +# ===== Positional match (cars[N]) ===== +# ========================================== + +# START NestedPositional +# "the FIRST car has make = Toyota" — Doc 3's first car is Honda, so it's excluded +response = docs.query.fetch_objects( + # highlight-start + filters=Filter.by_property("cars[0].make").equal("Toyota"), + # highlight-end + return_properties=["title"], +) +# END NestedPositional + +assert len(response.objects) == 2 + + +# ========================================== +# ===== Same-element AND across leaves ===== +# ========================================== + +# START NestedSameElementAnd +# "the SAME car is both Toyota AND red" — only Doc 1's first car qualifies. +# Without same-element correlation a doc with separate (Toyota, blue) and +# (Honda, red) cars would also match, which is wrong. +response = docs.query.fetch_objects( + # highlight-start + filters=( + Filter.by_property("cars.make").equal("Toyota") + & Filter.by_property("cars.color").equal("red") + ), + # highlight-end + return_properties=["title"], +) +# END NestedSameElementAnd + +assert len(response.objects) == 1 + + +# ========================================== +# ===== Recursive path (object[] inside object[]) ===== +# ========================================== + +# START NestedRecursive +# "any tire on any car is wider than 200" — Doc 1 (215) and Doc 3 (250) +response = docs.query.fetch_objects( + # highlight-start + filters=Filter.by_property("cars.tires.width").greater_than(200), + # highlight-end + return_properties=["title"], +) +# END NestedRecursive + +assert len(response.objects) == 2 + + +# ========================================== +# ===== IsNull on an intermediate object ===== +# ========================================== + +# START NestedIsNull +# "the first car has no tires" — only the Toyota in Doc 2 +response = docs.query.fetch_objects( + # highlight-start + filters=Filter.by_property("cars[0].tires").is_none(True), + # highlight-end + return_properties=["title"], +) +# END NestedIsNull + +assert len(response.objects) == 1 + + +client.collections.delete("Document") +client.close() diff --git a/_includes/code/java-v6/src/test/java/ManageCollectionsTest.java b/_includes/code/java-v6/src/test/java/ManageCollectionsTest.java index f54dd8cef..62cf717d4 100644 --- a/_includes/code/java-v6/src/test/java/ManageCollectionsTest.java +++ b/_includes/code/java-v6/src/test/java/ManageCollectionsTest.java @@ -408,12 +408,13 @@ void testReplicationSettings() throws IOException { @Test void testAsyncRepair() throws IOException { // START AsyncRepair + // Async replication runs by default when the replication factor is greater than 1 client.collections.create("Article", col -> col.replication( - Replication.of(rep -> rep.replicationFactor(1).asyncEnabled(true)))); + Replication.of(rep -> rep.replicationFactor(1)))); // END AsyncRepair var config = client.collections.getConfig("Article").get(); - assertThat(config.replication().asyncEnabled()).isTrue(); + assertThat(config.replication().replicationFactor()).isEqualTo(1); } @Test @@ -421,7 +422,6 @@ void testAllReplicationSettings() throws IOException { // START AllReplicationSettings threeNodeClient.collections.create("Article", col -> col.replication(Replication.of(rep -> rep.replicationFactor(3) - .asyncEnabled(true) .deletionStrategy(DeletionStrategy.TIME_BASED_RESOLUTION) .asyncReplication(AsyncReplicationConfig.of(async -> async .propagationConcurrency(5) @@ -431,7 +431,6 @@ void testAllReplicationSettings() throws IOException { var config = threeNodeClient.collections.getConfig("Article").get(); assertThat(config.replication().replicationFactor()).isEqualTo(3); - assertThat(config.replication().asyncEnabled()).isTrue(); // START UpdateReplicationSettings var collection = threeNodeClient.collections.use("Article"); diff --git a/_includes/collection-mutable-parameters.mdx b/_includes/collection-mutable-parameters.mdx index 67fca22d9..9f177f235 100644 --- a/_includes/collection-mutable-parameters.mdx +++ b/_includes/collection-mutable-parameters.mdx @@ -14,7 +14,6 @@ - `autoTenantCreation` (introduced in `v1.25.0`) - `autoTenantActivation` (introduced in `v1.25.2`) - `replicationConfig` - - `asyncEnabled` (introduced in `v1.26.0`) - `factor` (not mutable in `v1.25` or higher) - `deletionStrategy` (introduced in `v1.27.0`) - `vectorIndexConfig` diff --git a/_includes/feature-notes/async-config-collection.mdx b/_includes/feature-notes/async-config-collection.mdx index fbee348d5..bf705bf63 100644 --- a/_includes/feature-notes/async-config-collection.mdx +++ b/_includes/feature-notes/async-config-collection.mdx @@ -1,3 +1,3 @@ :::info Added in `v1.36` -These per-collection parameters override the corresponding [environment variables](/deploy/configuration/async-rep) at the collection level. If not set, the collection inherits the cluster-wide environment variable defaults. +The corresponding cluster-wide [environment variables](/deploy/configuration/async-rep) override these per-collection parameters. ::: diff --git a/_includes/feature-notes/boost.mdx b/_includes/feature-notes/boost.mdx new file mode 100644 index 000000000..68167df1d --- /dev/null +++ b/_includes/feature-notes/boost.mdx @@ -0,0 +1,3 @@ +:::caution Preview — added in `v1.38` +This is a preview feature. The API may change in future releases. +::: diff --git a/_maintenance/removed-content-cleanup.md b/_maintenance/removed-content-cleanup.md new file mode 100644 index 000000000..ecd50c81d --- /dev/null +++ b/_maintenance/removed-content-cleanup.md @@ -0,0 +1,27 @@ +# Removed content — cleanup tracking + +This file tracks documentation content that describes **removed** Weaviate features, +environment variables, or configuration fields. Rather than deleting such content the +moment a feature is removed, we keep it in place with a short "Removed in `vX.Y`" note so +that users upgrading from an older version can still find the entry and understand what +happened to it. Once enough releases have passed that few users are upgrading across the +removal boundary, the noted content should be deleted. + +## Policy + +- When a feature/env var/config field is removed from Weaviate, **mark** the corresponding + docs entry as `Removed in vX.Y` instead of deleting it immediately. +- Add a row to the table below so the kept-but-stale content can be found and cleaned later. +- **Suggested cleanup window:** keep the note for roughly three minor releases after the + removal, then delete the entry and remove its row here. (Adjust per the supported-version + policy at cleanup time — these are guidelines, not hard commitments.) + +## Tracked entries + +| Page / file | Removed item | Removed in | Suggested cleanup | Notes | +| --- | --- | --- | --- | --- | +| `docs/deploy/configuration/env-vars/index.md` | `ASYNC_REPLICATION_CLUSTER_MAX_WORKERS` (table row) | `v1.38` | `v1.41`+ | Replaced by `ASYNC_REPLICATION_SCHEDULER_WORKERS`. | +| `docs/deploy/configuration/env-vars/index.md` | `ASYNC_REPLICATION_ALIVE_NODES_CHECKING_FREQUENCY` (table row) | `v1.38` | `v1.41`+ | Scheduler no longer polls alive nodes separately. | +| `docs/deploy/configuration/env-vars/runtime-config.md` | `async_replication_cluster_max_workers` (override mapping row) | `v1.38` | `v1.41`+ | Runtime override removed alongside the env var. | +| `docs/deploy/configuration/async-rep.md` | "Removed environment variables (v1.38)" `
` block (both vars above) | `v1.38` | `v1.41`+ | Delete the whole block at cleanup. | +| `docs/deploy/configuration/replication.md` | "Removed in `v1.38`" `:::note` admonition | `v1.38` | `v1.41`+ | Mentions both removed env vars. | diff --git a/docs/deploy/configuration/async-rep.md b/docs/deploy/configuration/async-rep.md index cead8e7c6..1b109c410 100644 --- a/docs/deploy/configuration/async-rep.md +++ b/docs/deploy/configuration/async-rep.md @@ -16,7 +16,8 @@ This applies solely to data objects, as metadata consistency is treated differen ### Under the Hood - Async replication operates as a background process either per tenant (in a multi-tenant collection) or per shard (in a non-multi-tenant collection). -- It is disabled by default but can be enabled through collection configuration changes, similar to setting the replication factor. +- As of Weaviate `v1.38`, async replication is **enabled by default** for any collection with a replication factor greater than `1`. There is no longer a per-collection toggle to enable or disable it; the per-collection `asyncConfig` object is used only to fine-tune the behavior of an already-running process. +- To turn async replication off across the entire cluster, set the [`ASYNC_REPLICATION_DISABLED`](#async_replication_disabled) environment variable to `true`. ## Environment Variable Deep Dive @@ -48,13 +49,36 @@ Globally disables the entire async replication feature.
Cluster Worker Limits -#### `ASYNC_REPLICATION_CLUSTER_MAX_WORKERS` -Sets the maximum number of concurrent async replication workers across the entire cluster. +#### `ASYNC_REPLICATION_SCHEDULER_WORKERS` +Sets the number of workers in the cluster-wide pool that the async replication scheduler uses to run hashbeat work across all shards and tenants. -- Its default value is `30`. -- **Use case**: Limits the total number of concurrent replication workers to prevent resource exhaustion in large clusters with many collections or tenants. +- Its default value is `10`. The maximum is `100`. +- **Use case**: A single bounded worker pool replaces the previous per-shard goroutines, so this is the main lever for capping async replication's total concurrency and preventing resource exhaustion on clusters with many collections or tenants. - **Special Considerations**: - - This is a cluster-wide cap. Individual collections can set their own `maxWorkers` via the per-collection [`asyncConfig`](/weaviate/config-refs/collections#async-config), but the total across all collections will not exceed this cluster limit. + - This is a cluster-wide setting. There is no per-collection worker count; collections share this single pool. + +:::note Changed in `v1.38` +`ASYNC_REPLICATION_SCHEDULER_WORKERS` replaces the removed `ASYNC_REPLICATION_CLUSTER_MAX_WORKERS` environment variable, and the per-collection `maxWorkers` option has been removed. +::: + +#### `ASYNC_REPLICATION_HASHTREE_INIT_CONCURRENCY` +Sets how many shards may initialize (build) their hash tree concurrently when async replication starts up. + +- Its default value is `100`. +- **Use case**: Bounds the burst of work when many shards begin async replication at once, for example after a node restart or when many replicated collections exist. + +
+ +
+ Removed environment variables (v1.38) + +These variables were removed when async replication moved to a centralized scheduler in `v1.38`. They are listed here for reference and are no longer read by Weaviate. + +#### `ASYNC_REPLICATION_CLUSTER_MAX_WORKERS` +**Removed in `v1.38`.** Previously set the maximum number of concurrent async replication workers across the cluster (default `30`). Replaced by [`ASYNC_REPLICATION_SCHEDULER_WORKERS`](#async_replication_scheduler_workers). + +#### `ASYNC_REPLICATION_ALIVE_NODES_CHECKING_FREQUENCY` +**Removed in `v1.38`.** Previously defined how often the background process checked for changes in node availability (default `5s`). The scheduler no longer uses a separate alive-nodes polling mechanism.
@@ -153,16 +177,6 @@ Defines a shorter frequency for subsequent comparison and propagation attempts w
-
- Node Status Monitoring - -#### `ASYNC_REPLICATION_ALIVE_NODES_CHECKING_FREQUENCY` -Defines the frequency at which the system checks for changes in the availability of nodes within the cluster. - - Its default value is `5s`. The value requires a time unit suffix (e.g. `5s`, `1m`). - - **Use Case(s)**: When a node rejoins the cluster after a period of downtime, it is highly likely to be out of sync. This setting ensures that the replication process is initiated promptly. - -
-
Timeout Management diff --git a/docs/deploy/configuration/env-vars/index.md b/docs/deploy/configuration/env-vars/index.md index 3db54c30e..1a07eca4f 100644 --- a/docs/deploy/configuration/env-vars/index.md +++ b/docs/deploy/configuration/env-vars/index.md @@ -67,8 +67,8 @@ import APITable from '@site/src/components/APITable'; | `MAXIMUM_CONCURRENT_BUCKET_LOADS` | Maximum number of buckets that can be loaded concurrently during startup. This is a safeguard to prevent overwhelming the operating system when loading large numbers of collections. Default: `100`
Added in `v1.31.22` | `string - number` | `50` | | `MAXIMUM_CONCURRENT_SHARD_LOADS` | Maximum number of shards that can be loaded concurrently during startup. This is a safeguard to prevent overwhelming the operating system when loading large numbers of collections. Default: `100` | `string - number` | `50` | | `MCP_SERVER_CONFIG_PATH` | Path to a YAML file for customizing MCP tool descriptions. Useful for prompt engineering the LLM's understanding of your specific data. If not provided or file malformed, the default descriptions will be used. Default: `""` (default tool descriptions from the [source code](https://github.com/weaviate/weaviate/tree/main/adapters/handlers/mcp) will be used)
Added in `v1.37.1` | `string` | `/etc/weaviate/mcp-config.yaml` | -| `MCP_SERVER_ENABLED` | Enable the built-in MCP server. When enabled, the MCP endpoint is available at `/v1/mcp` on the REST API port. Default: `false`
Added in `v1.37.1` | `boolean` | `true` | -| `MCP_SERVER_WRITE_ACCESS_ENABLED` | Enable write tools (`weaviate-objects-upsert`) on the MCP server. When `false`, only read and query tools are available. Default: `false`
Added in `v1.37.1` | `boolean` | `true` | +| `MCP_SERVER_ENABLED` | Enable the built-in MCP server. When enabled, the MCP endpoint is available at `/v1/mcp` on the REST API port. Default: `false`
Added in `v1.37.1`. [Runtime-configurable](./runtime-config.md#mcp) from `v1.38`. | `boolean` | `true` | +| `MCP_SERVER_WRITE_ACCESS_ENABLED` | Enable write tools (`weaviate-objects-upsert`) on the MCP server. When `false`, only read and query tools are available. Default: `false`
Added in `v1.37.1`. [Runtime-configurable](./runtime-config.md#mcp) from `v1.38`. | `boolean` | `true` | | `MEMORY_READONLY_PERCENTAGE` | If memory usage is higher than the given percentage all shards on the affected node will be marked as `READONLY`, meaning all future write requests will fail. (Default: `0` - i.e. no limit) | `string - number` | `75` | | `MEMORY_WARNING_PERCENTAGE` | If memory usage is higher than the given percentage a warning will be logged by all shards on the affected node's disk. (Default: `0` - i.e. no limit) | `string - number` | `85` | | `MODULES_CLIENT_TIMEOUT` | Timeout for requests to Weaviate modules. Default: `50s` | `string - duration` | `5s`, `10m`, `1h` | @@ -90,6 +90,7 @@ import APITable from '@site/src/components/APITable'; | `PERSISTENCE_LSM_MAX_SEGMENT_SIZE` | Maximum size of a segment in the [LSM store](/weaviate/concepts/storage.md#object-and-inverted-index-store). Set this to limit disk usage spikes during compaction to ~2x the segment size. Default: no limit | `string` | `4GiB` (IEC units), `4GB` (SI units), `4000000000` (bytes) | | `PROMETHEUS_MONITORING_ENABLED` | If set, Weaviate collects [metrics in a Prometheus-compatible format](/deploy/configuration/monitoring.md) | `boolean` | `false` | | `PROMETHEUS_MONITORING_GROUP` | If set, Weaviate groups metrics for the same class across all shards. | `boolean` | `true` | +| `QUERY_BOOST_DEFAULT_DEPTH` | Default candidate-pool size used when a [Boost](/weaviate/search/boost.md) query does not set its own `depth`. The primary search retrieves this many candidates before the boost rescorer runs. Must be a positive integer and is hard-capped by `QUERY_MAXIMUM_RESULTS`. Default: `100`
Added in `v1.38` | `string - number` | `200` | | `QUERY_CROSS_REFERENCE_DEPTH_LIMIT` | Sets the maximum depth of cross-references to be resolved in a query. Defaults to 5. | `string - number` | `3` | | `QUERY_DEFAULTS_LIMIT` | Sets the default number of objects to be returned in a query. | `string - number` | `25`
Defaults to `10`| | `QUERY_MAXIMUM_RESULTS` | Sets the maximum total number of objects that can be retrieved. | `string - number` | `10000` | @@ -225,9 +226,7 @@ For more information on authentication and authorization, see the [Authenticatio | `RAFT_BOOTSTRAP_EXPECT` | The number of voter notes at bootstrapping time | `string - number` | `1` | | `RAFT_BOOTSTRAP_TIMEOUT` | The time in seconds to wait for the cluster to bootstrap | `string - number` | `90` | | `RAFT_DRAIN_SLEEP` | Grace period before shutdown to allow ongoing operations to complete. (Default: `200ms`) | `string - number` | `2s` | -| `RAFT_ENABLE_FQDN_RESOLVER` | If `true`, use DNS lookup instead of memberlist lookup for Raft. Removed in `v1.30`. ([Read more](/weaviate/concepts/cluster.md#node-discovery)) | `boolean` | `true` | | `RAFT_ENABLE_ONE_NODE_RECOVERY` | Enable running the single node recovery routine on restart. This is useful if the default hostname has changed and a single node cluster believes there are supposed to be two nodes. | `boolean` | `false` | -| `RAFT_FQDN_RESOLVER_TLD` | The top-level domain to use for DNS lookup, in `[node-id].[tld]` format. Removed in `v1.30`. ([Read more](/weaviate/concepts/cluster.md#node-discovery)) | `string` | `example.com` | | `RAFT_GRPC_MESSAGE_MAX_SIZE` | The maximum internal raft gRPC message size in bytes. Defaults to 1073741824 | `string - number` | `1073741824` | | `RAFT_JOIN` | Manually set Raft voter nodes. If set, RAFT_BOOTSTRAP_EXPECT needs to be adjusted manually to match the number of Raft voters. | `string` | `weaviate-0,weaviate-1` | | `RAFT_METADATA_ONLY_VOTERS` | If `true`, voter nodes only handle the schema. They do not accept any data. | `boolean` | `false` | @@ -250,12 +249,14 @@ For more information on authentication and authorization, see the [Authenticatio | Variable | Description | Type | Example Value | | --- | --- | --- | --- | -| `ASYNC_REPLICATION_DISABLED` | Disable async replication. Default: `false` | `boolean` | `false` | -| `ASYNC_REPLICATION_CLUSTER_MAX_WORKERS` | Maximum concurrent async replication workers across the cluster. Default: `30` | `string - number` | `10` | +| `ASYNC_REPLICATION_DISABLED` | Disable async replication cluster-wide. When `false` (default), async replication runs automatically for any collection with a replication factor greater than `1`. Default: `false` | `boolean` | `false` | +| `ASYNC_REPLICATION_SCHEDULER_WORKERS` | Number of workers in the cluster-wide pool that run async replication work across all shards and tenants. Added in `v1.38`, replacing `ASYNC_REPLICATION_CLUSTER_MAX_WORKERS`. Default: `10`, Max: `100`
[Read more.](/deploy/configuration/async-rep.md#async_replication_scheduler_workers) | `string - number` | `10` | +| `ASYNC_REPLICATION_HASHTREE_INIT_CONCURRENCY` | Number of shards that may build their hash tree concurrently when async replication starts up. Added in `v1.38`. Default: `100`
[Read more.](/deploy/configuration/async-rep.md#async_replication_hashtree_init_concurrency) | `string - number` | `100` | +| `ASYNC_REPLICATION_CLUSTER_MAX_WORKERS` | **Removed in `v1.38`.** Previously set the maximum number of concurrent async replication workers across the cluster. Replaced by `ASYNC_REPLICATION_SCHEDULER_WORKERS`. | `string - number` | `30` | | `ASYNC_REPLICATION_HASHTREE_HEIGHT` | Height of the hash tree used for data comparison between nodes. If the height is `0` each node will store just one digest per shard. Default: `16` (single-tenant) / `10` (multi-tenant), Min: `0`, Max: `20`
[Read more about potentially increased memory consumption.](/weaviate/concepts/replication-architecture/consistency#memory-and-performance-considerations-for-async-replication) | `string - number` | `10` | | `ASYNC_REPLICATION_FREQUENCY` | Frequency of periodic data comparison between nodes. Default: `30s` | `string - duration` | `60s` | | `ASYNC_REPLICATION_FREQUENCY_WHILE_PROPAGATING` | Frequency of data comparison between nodes while propagation is active. Default: `3s` | `string - duration` | `5s` | -| `ASYNC_REPLICATION_ALIVE_NODES_CHECKING_FREQUENCY` | Frequency of how often the background process checks for changes in the availability of nodes. Default: `5s` | `string - duration` | `20s` | +| `ASYNC_REPLICATION_ALIVE_NODES_CHECKING_FREQUENCY` | **Removed in `v1.38`.** Previously set how often the background process checked for changes in node availability. No longer used by the async replication scheduler. | `string - duration` | `5s` | | `ASYNC_REPLICATION_LOGGING_FREQUENCY` | Frequency of how often the background process logs any events. Default: `60s` | `string - duration` | `7s` | | `ASYNC_REPLICATION_DIFF_BATCH_SIZE` | Specifies the batch size for comparing digest information between nodes. Default: `1000`, Min: `1`, Max: `10000` |`string - number` | `2000` | | `ASYNC_REPLICATION_DIFF_PER_NODE_TIMEOUT` | Defines the time limit a node has to provide a comparison response. Default: `10s` | `string - duration` | `30s` | diff --git a/docs/deploy/configuration/env-vars/runtime-config.md b/docs/deploy/configuration/env-vars/runtime-config.md index f0c294cdf..69c118e55 100644 --- a/docs/deploy/configuration/env-vars/runtime-config.md +++ b/docs/deploy/configuration/env-vars/runtime-config.md @@ -57,7 +57,9 @@ The following overrides are currently supported: | Runtime override name | Environment variable name | | :----------------------------------------------- | :------------------------------------------- | | `async_replication_disabled` | `ASYNC_REPLICATION_DISABLED` | -| `async_replication_cluster_max_workers` | `ASYNC_REPLICATION_CLUSTER_MAX_WORKERS` | +| `async_replication_scheduler_workers` | `ASYNC_REPLICATION_SCHEDULER_WORKERS` | +| `async_replication_hashtree_init_concurrency` | `ASYNC_REPLICATION_HASHTREE_INIT_CONCURRENCY`| +| `async_replication_cluster_max_workers` _(removed in `v1.38`)_ | `ASYNC_REPLICATION_CLUSTER_MAX_WORKERS` _(removed in `v1.38`)_ | | `autoschema_enabled` | `AUTOSCHEMA_ENABLED` | | `default_quantization` | `DEFAULT_QUANTIZATION` | | `default_sharding_count` | `DEFAULT_SHARDING_COUNT` | @@ -114,6 +116,17 @@ The following overrides are currently supported: | `authentication_oidc_skip_client_id_check` | `AUTHENTICATION_OIDC_SKIP_CLIENT_ID_CHECK` | | `authentication_oidc_username_claim` | `AUTHENTICATION_OIDC_USERNAME_CLAIM` | +### MCP + +Added in `v1.38`. Toggling these at runtime does not require a cluster restart — the HTTP handlers stay registered and per-request checks pick up the new value. See [MCP server — Toggle without restart](/weaviate/configuration/mcp-server.mdx#toggle-without-restart) for behavior details. + +| Runtime override name | Environment variable name | +| :-------------------------------- | :---------------------------------- | +| `mcp_server_enabled` | `MCP_SERVER_ENABLED` | +| `mcp_server_write_access_enabled` | `MCP_SERVER_WRITE_ACCESS_ENABLED` | + +`MCP_SERVER_CONFIG_PATH` is intentionally **not** runtime-configurable — tool descriptions are baked into the tool schemas at registration. + Refer to the [Environment variables](./index.md) page for descriptions on each configuration option ## Operation and monitoring diff --git a/docs/deploy/configuration/horizontal-scaling.mdx b/docs/deploy/configuration/horizontal-scaling.mdx index 2b6b9c44b..e1e8a64c5 100644 --- a/docs/deploy/configuration/horizontal-scaling.mdx +++ b/docs/deploy/configuration/horizontal-scaling.mdx @@ -338,7 +338,6 @@ curl \ ], "replicationConfig": { "factor": 3, - "asyncEnabled": true, "deletionStrategy": "TimeBasedResolution" } }' \ diff --git a/docs/deploy/configuration/monitoring.md b/docs/deploy/configuration/monitoring.md index a954c13f7..4ecf2dc15 100644 --- a/docs/deploy/configuration/monitoring.md +++ b/docs/deploy/configuration/monitoring.md @@ -64,7 +64,8 @@ Be aware that metrics do not follow the semantic versioning guidelines of other ::: -The list of metrics that are obtainable through Weaviate's metric system is constantly being expanded. The complete list of metrics can be found in the source code files: +The list of metrics that are obtainable through Weaviate's metric system is constantly being expanded. The complete list of metrics can be found in the source code files: + - [`usecases/monitoring/prometheus.go`](https://github.com/weaviate/weaviate/blob/main/usecases/monitoring/prometheus.go) - [`usecases/replica/metrics.go`](https://github.com/weaviate/weaviate/blob/main/usecases/replica/metrics.go) - [`adapters/repos/db/metrics.go`](https://github.com/weaviate/weaviate/blob/main/adapters/repos/db/metrics.go) @@ -270,7 +271,7 @@ These metrics track Write-Ahead Log (WAL) recovery operations during startup. | `lsm_bucket_wal_recovery_failure_count` | Number of failed LSM bucket WAL recoveries | `strategy` | `Counter` | | `lsm_bucket_wal_recovery_duration_seconds` | Duration of LSM bucket WAL recovery in seconds | `strategy` | `Histogram` | -### Schema & cluster consensus +### Schema & cluster consensus #### Schema & RAFT consensus @@ -356,17 +357,17 @@ These metrics track Write-Ahead Log (WAL) recovery operations during startup. #### Backup & restore -| Metric | Description | Labels | Type | -| --------------------------------- | --------------------------------------------------------- | ---------------------------- | --------- | -| `backup_restore_ms` | Duration of a backup restore | `backend_name`, `class_name` | `Summary` | -| `backup_restore_class_ms` | Duration restoring class | `class_name` | `Summary` | -| `backup_restore_init_ms` | Startup phase of a backup restore | `backend_name`, `class_name` | `Summary` | -| `backup_restore_from_backend_ms` | File transfer stage of a backup restore | `backend_name`, `class_name` | `Summary` | -| `backup_store_to_backend_ms` | File transfer stage of a backup store | `backend_name`, `class_name` | `Summary` | -| `bucket_pause_durations_ms` | Bucket pause durations | `bucket_dir` | `Summary` | -| `backup_restore_data_transferred` | Total number of bytes transferred during a backup restore | `backend_name`, `class_name` | `Counter` | -| `backup_store_data_transferred` | Total number of bytes transferred during a backup store | `backend_name`, `class_name` | `Counter` | -| `weaviate_restore_phase_duration_seconds` | Duration of restore phases (prepare, object_storage_download, schema_apply) | `phase` | `Histogram` | +| Metric | Description | Labels | Type | +| ----------------------------------------- | --------------------------------------------------------------------------- | ---------------------------- | ----------- | +| `backup_restore_ms` | Duration of a backup restore | `backend_name`, `class_name` | `Summary` | +| `backup_restore_class_ms` | Duration restoring class | `class_name` | `Summary` | +| `backup_restore_init_ms` | Startup phase of a backup restore | `backend_name`, `class_name` | `Summary` | +| `backup_restore_from_backend_ms` | File transfer stage of a backup restore | `backend_name`, `class_name` | `Summary` | +| `backup_store_to_backend_ms` | File transfer stage of a backup store | `backend_name`, `class_name` | `Summary` | +| `bucket_pause_durations_ms` | Bucket pause durations | `bucket_dir` | `Summary` | +| `backup_restore_data_transferred` | Total number of bytes transferred during a backup restore | `backend_name`, `class_name` | `Counter` | +| `backup_store_data_transferred` | Total number of bytes transferred during a backup store | `backend_name`, `class_name` | `Counter` | +| `weaviate_restore_phase_duration_seconds` | Duration of restore phases (prepare, object_storage_download, schema_apply) | `phase` | `Histogram` | #### Shard management @@ -438,9 +439,16 @@ These metrics track Write-Ahead Log (WAL) recovery operations during startup. These metrics track asynchronous replication operations for maintaining data consistency across replicas. +:::note Changed in `v1.38` +As of Weaviate `v1.38`, async replication runs through a centralized scheduler with a bounded worker pool (replacing the previous per-shard goroutines). The `async_replication_scheduler_*` metrics below are new in `v1.38`, and the `async_replication_goroutines_running` metric has been removed. +::: + | Metric | Description | Labels | Type | | -------------------------------------------------------- | ------------------------------------------------------------------------ | ------------------------------------- | ----------- | -| `async_replication_goroutines_running` | Number of currently running async replication goroutines | `type` (hashbeater, hashbeat_trigger) | `Gauge` | +| `async_replication_scheduler_worker_pool_size` | Current target size of the scheduler worker pool | None | `Gauge` | +| `async_replication_scheduler_workers_active` | Number of scheduler worker goroutines currently executing a hashbeat cycle | None | `Gauge` | +| `async_replication_scheduler_shards_registered` | Number of shards currently registered with the async replication scheduler | None | `Gauge` | +| `async_replication_scheduler_queue_depth` | Number of shards waiting in the scheduler heap (not in-flight) | None | `Gauge` | | `async_replication_hashtree_init_count` | Count of async replication hashtree initializations | None | `Counter` | | `async_replication_hashtree_init_running` | Number of currently running hashtree initializations | None | `Gauge` | | `async_replication_hashtree_init_failure_count` | Count of async replication hashtree initialization failures | None | `Counter` | @@ -451,10 +459,13 @@ These metrics track asynchronous replication operations for maintaining data con | `async_replication_iteration_running` | Number of currently running async replication iterations | None | `Gauge` | | `async_replication_hashtree_diff_duration_seconds` | Duration of async replication hashtree diff computation in seconds | None | `Histogram` | | `async_replication_object_digests_diff_duration_seconds` | Duration of async replication object digests diff computation in seconds | None | `Histogram` | +| `async_replication_objects_diff_total` | Total objects found in diff per hashbeat cycle (queued for propagation or locally deleted before propagation) | None | `Counter` | | `async_replication_propagation_count` | Count of async replication propagation executions | None | `Counter` | | `async_replication_propagation_failure_count` | Count of async replication propagation failures | None | `Counter` | | `async_replication_propagation_object_count` | Count of objects propagated by async replication | None | `Counter` | | `async_replication_propagation_duration_seconds` | Duration of async replication propagation in seconds | None | `Histogram` | +| `async_replication_local_deletions_total` | Total local object deletions applied due to remote-deleted verdicts or deletion-conflict responses during async replication | None | `Counter` | +| `async_replication_reconcile_failures_total` | Number of indices that failed to reconcile async replication with the global `ASYNC_REPLICATION_DISABLED` flag | None | `Counter` | #### Replication coordinator @@ -474,6 +485,26 @@ These metrics track the replication coordinator's read and write operations acro | `replication_coordinator_reads_duration_seconds` | Duration in seconds of read operations from replicas | None | `Histogram` | | `replication_read_repair_duration_seconds` | Duration in seconds of read repair operations | None | `Histogram` | +### MCP server + +Added in `v1.38`. These metrics track tool traffic, latency, auth failures, and the live state of the runtime write-access flag for the built-in [Weaviate MCP server](/weaviate/configuration/mcp-server.mdx). + +| Metric | Description | Labels | Type | +| ----------------------------------------- | ------------------------------------------------------------------------------------------------- | ---------------- | ----------- | +| `weaviate_mcp_tool_calls_total` | Total MCP tool invocations. | `tool`, `status` | `Counter` | +| `weaviate_mcp_tool_call_duration_seconds` | Latency of MCP tool calls (LatencyBuckets histogram). | `tool`, `status` | `Histogram` | +| `weaviate_mcp_tool_calls_inflight` | In-flight MCP tool calls per tool. Catches one slow tool starving the rest. | `tool` | `Gauge` | +| `weaviate_mcp_auth_failures_total` | MCP authentication and authorization failures. | `reason` | `Counter` | +| `weaviate_mcp_tools_listed_total` | `tools/list` calls, labeled with whether the write tool was visible in the response. | `write_access` | `Counter` | +| `weaviate_mcp_write_access_enabled` | Live state of `MCP_SERVER_WRITE_ACCESS_ENABLED`, polled at scrape time. Reflects runtime toggles. | None | `Gauge` | + +Label values: + +- **`tool`** — the MCP tool name (e.g. `weaviate-query-hybrid`, `weaviate-objects-upsert`). +- **`status`** — `success` · `error` · `denied` · `write_disabled`. `denied` covers authorization failures classified via the `Forbidden` / `Unauthenticated` error families. `write_disabled` is emitted when a write call hits the runtime guard. +- **`reason`** — `missing_token` · `invalid_token` · `forbidden` · `unauthenticated`. `missing_token` and `invalid_token` are detected at the principal-extraction step; `forbidden` and `unauthenticated` are detected at authorization time. +- **`write_access`** — `enabled` / `disabled`, matching the live state of `MCP_SERVER_WRITE_ACCESS_ENABLED` at the time of the `tools/list` call. + --- diff --git a/docs/deploy/configuration/replication.md b/docs/deploy/configuration/replication.md index c1cfca1ba..989a1ed99 100644 --- a/docs/deploy/configuration/replication.md +++ b/docs/deploy/configuration/replication.md @@ -48,7 +48,7 @@ When Weaviate detects inconsistent data across nodes, it attempts to repair the Weaviate offers [async replication](/weaviate/concepts/replication-architecture/consistency.md#async-replication) to proactively detect inconsistencies. In earlier versions, Weaviate uses a [repair-on-read](/weaviate/concepts/replication-architecture/consistency.md#repair-on-read) strategy to repair inconsistencies at read time. -Repair-on-read is automatic. To activate async replication, set `asyncEnabled` to true in the `replicationConfig` section of your collection definition. +Repair-on-read is automatic. As of Weaviate `v1.38`, async replication is also **enabled by default** for any collection with a replication factor greater than `1` — there is no longer a per-collection flag to switch it on. To turn it off cluster-wide, set the [`ASYNC_REPLICATION_DISABLED`](/deploy/configuration/env-vars/index.md#async-replication) environment variable to `true`. The `replicationConfig` section is used to set the replication factor and to fine-tune async replication via `asyncConfig`: import ReplicationConfigWithAsyncRepair from '/\_includes/code/configuration/replication-consistency.mdx'; @@ -64,8 +64,10 @@ Update the following [environment variables](/deploy/configuration/env-vars/inde #### Worker limits -- **Set the cluster-wide worker cap:** `ASYNC_REPLICATION_CLUSTER_MAX_WORKERS` - Set the maximum number of concurrent async replication workers across the entire cluster. Default: `30`. Individual collections can set their own `maxWorkers` via `asyncConfig`, but the total across all collections will not exceed this cluster limit. +- **Set the scheduler worker pool size:** `ASYNC_REPLICATION_SCHEDULER_WORKERS` + Set the number of workers in the cluster-wide pool that run async replication work across all shards and tenants. Default: `10`, maximum: `100`. As of `v1.38` this replaces the removed `ASYNC_REPLICATION_CLUSTER_MAX_WORKERS` variable and the per-collection `maxWorkers` option; collections share this single pool. +- **Set hash tree init concurrency:** `ASYNC_REPLICATION_HASHTREE_INIT_CONCURRENCY` + Set how many shards may build their hash tree concurrently when async replication starts up. Default: `100`. #### Logging @@ -78,13 +80,15 @@ Update the following [environment variables](/deploy/configuration/env-vars/inde Define how often each node compares its local data with other nodes. - **Set comparison timeout:** `ASYNC_REPLICATION_DIFF_PER_NODE_TIMEOUT` Optionally configure a timeout for how long to wait during comparison when a node is unresponsive. -- **Monitor node availability:** `ASYNC_REPLICATION_ALIVE_NODES_CHECKING_FREQUENCY` - Trigger comparisons whenever there’s a change in node availability. - **Configure hash tree height:** `ASYNC_REPLICATION_HASHTREE_HEIGHT` Specify the size of the hash tree, which helps narrow down data differences by comparing hash digests at multiple levels instead of scanning entire datasets. See [this page](/weaviate/concepts/replication-architecture/consistency.md#memory-and-performance-considerations-for-async-replication) for more information on the memory and performance considerations for async replication. - **Batch size for digest comparison:** `ASYNC_REPLICATION_DIFF_BATCH_SIZE` Define the number of objects whose digest (e.g., last update time) is compared between nodes before propagating actual objects. +:::note Removed in `v1.38` +The `ASYNC_REPLICATION_CLUSTER_MAX_WORKERS` (replaced by `ASYNC_REPLICATION_SCHEDULER_WORKERS`) and `ASYNC_REPLICATION_ALIVE_NODES_CHECKING_FREQUENCY` environment variables were removed in `v1.38` when async replication moved to a centralized scheduler. +::: + #### Data synchronization Once differences between nodes are detected, Weaviate propagates outdated or missing data. Configure synchronization as follows: diff --git a/docs/weaviate/api/graphql/filters.md b/docs/weaviate/api/graphql/filters.md index 855339515..7055aa676 100644 --- a/docs/weaviate/api/graphql/filters.md +++ b/docs/weaviate/api/graphql/filters.md @@ -149,7 +149,7 @@ Starting with `v1.12.0` you can configure your own [stopword lists for the inver ## Multiple operands -You can set multiple operands or [nest conditions](../../search/filters.md#nested-filters). +You can set multiple operands or [combine conditions with `And` / `Or`](../../search/filters.md#combine-filters-with-and-or-or). :::tip You can filter datetimes similarly to numbers, with the `valueDate` given as `string` in [RFC3339](https://datatracker.ietf.org/doc/rfc3339/) format. @@ -472,6 +472,61 @@ import GraphQLFiltersWhereBeaconCount from '/_includes/code/graphql.filters.wher
+### By nested object property + +:::caution Preview feature + +Available from Weaviate `v1.38` as a preview, gated by `WEAVIATE_PREVIEW_NESTED_FILTERING=on` on the server. See [Filter on nested object properties](../../search/filters.md#filter-on-nested-object-properties) for the conceptual guide and worked examples. + +::: + +A `where` filter can target a leaf inside an [`object` / `object[]` property](../../config-refs/datatypes.md#object). The `path` is a **single-element array** containing a dotted path; `[N]` pins a segment to an array index. + +```graphql +# Any car has make = "Toyota" +{ + Get { + Document( + where: { + path: ["cars.make"] + operator: Equal + valueText: "Toyota" + } + ) { title } + } +} + +# The first car's third tire is a Bridgestone +{ + Get { + Document( + where: { + path: ["cars[0].tires[2].brand"] + operator: Equal + valueText: "Bridgestone" + } + ) { title } + } +} + +# Same-element correlation: the SAME car is both Toyota AND red +{ + Get { + Document( + where: { + operator: And + operands: [ + { path: ["cars.make"], operator: Equal, valueText: "Toyota" } + { path: ["cars.color"], operator: Equal, valueText: "red" } + ] + } + ) { title } + } +} +``` + +Don't confuse this with a [reference-path filter](#by-cross-references): a reference-path `path` has multiple elements traversing cross-references (`["inCity", "City", "name"]`), while a nested-path `path` is a **single element** with dots inside it (`["cars.make"]`). + ### By geo coordinates A special case of the `Where` filter is with geoCoordinates. This filter is only supported by the `Get{}` function. If you've set the `geoCoordinates` property type, you can search in an area based on kilometers. diff --git a/docs/weaviate/concepts/cluster.md b/docs/weaviate/concepts/cluster.md index a57e34cc8..788efba8c 100644 --- a/docs/weaviate/concepts/cluster.md +++ b/docs/weaviate/concepts/cluster.md @@ -112,54 +112,12 @@ By default, Weaviate nodes in a cluster use a gossip-like protocol through [Hash Weaviate - especially when running as a cluster - is optimized to run on Kubernetes. The [Weaviate Helm chart](/deploy/installation-guides/k8s-installation.md#weaviate-helm-chart) makes use of a `StatefulSet` and a headless `Service` that automatically configures node discovery. All you have to do is specify the desired node count. -
- FQDN for node discovery - -:::caution Removed in `v1.30` - -This was an experimental feature. Use with caution. - -::: - -There can be a situation where IP-address based node discovery is not optimal. In such cases, you can set `RAFT_ENABLE_FQDN_RESOLVER` and `RAFT_FQDN_RESOLVER_TLD` [environment variables](/deploy/configuration/env-vars/index.md#multi-node-instances) to enable fully qualified domain name (FQDN) based node discovery. - -If this feature is enabled, Weaviate uses the FQDN resolver to resolve the node name to the node IP address for metadata (e.g., Raft) communication. - -:::info FQDN: For metadata changes only -This feature is only used for metadata changes which [use Raft as the consensus mechanism](./replication-architecture/cluster-architecture.md#metadata-replication-raft). It does not affect data read/write operations. -::: - -#### Examples of when to use FQDN for node discovery - -The use of FQDN can resolve a situation where if IP addresses are re-used across different clusters, the nodes in one cluster could mistakenly discover nodes in another cluster. - -It can also be useful when using services (for example, Kubernetes) where the IP of the services is different from the actual node IP, but it proxies the connection to the node. - -#### Environment variables for FQDN node discovery - -`RAFT_ENABLE_FQDN_RESOLVER` is a Boolean flag. This flag enables the FQDN resolver. If set to `true`, Weaviate uses the FQDN resolver to resolve the node name to the node IP address. If set to `false`, Weaviate uses the memberlist lookup to resolve the node name to the node IP address. The default value is `false`. - -`RAFT_FQDN_RESOLVER_TLD` is a string that is appended in the format `[node-id].[tld]` when resolving a node-id to an IP address, where `[tld]` is the top-level domain. - -To use this feature, set `RAFT_ENABLE_FQDN_RESOLVER` to `true`. - -
- ## Node affinity of shards and/or replication shards Weaviate tries to select the node with the most available disk space. This only applies when creating a new class, rather than when adding more data to an existing single class. -
- Pre-v1.18.1 behavior - -In versions `v1.8.0`-`v1.18.0`, users could not specify the node-affinity of a specific shard or replication shard. - -Shards were assigned to 'live' nodes in a round-robin fashion starting with a random node. - -
- ## Consistency and current limitations * Weaviate adopts the [Raft consensus algorithm](https://raft.github.io/) which is a log-based algorithm coordinated by an elected leader. This brings an additional benefit in that concurrent schema changes are now supported.
If you are a Kubernetes user, see the [`1.25 migration guide`](/deploy/migration/weaviate-1-25.md) before you upgrade. To upgrade, you have to delete your existing StatefulSet. diff --git a/docs/weaviate/concepts/replication-architecture/cluster-architecture.md b/docs/weaviate/concepts/replication-architecture/cluster-architecture.md index 53ac6f211..0a9b08d2d 100644 --- a/docs/weaviate/concepts/replication-architecture/cluster-architecture.md +++ b/docs/weaviate/concepts/replication-architecture/cluster-architecture.md @@ -17,33 +17,6 @@ By default, Weaviate nodes in a cluster use a gossip-like protocol through [Hash Weaviate is optimized to run on Kubernetes, especially when operating as a cluster. The [Weaviate Helm chart](/deploy/installation-guides/k8s-installation.md#weaviate-helm-chart) makes use of a `StatefulSet` and a headless `Service` that automatically configures node discovery. -
- FQDN for node discovery - -There can be a situation where IP-address based node discovery is not optimal. In such cases, you can set `RAFT_ENABLE_FQDN_RESOLVER` and `RAFT_FQDN_RESOLVER_TLD` [environment variables](/deploy/configuration/env-vars/index.md#multi-node-instances) to enable [fully qualified domain name (FQDN)](https://en.wikipedia.org/wiki/Fully_qualified_domain_name) based node discovery. - -If this feature is enabled, Weaviate uses the FQDN resolver to resolve the node name to the node IP address for metadata (e.g., Raft) communication. - -:::info FQDN: For metadata changes only -This feature is only used for metadata changes which use Raft as the consensus mechanism. It does not affect data read/write operations. -::: - -#### Examples of when to use FQDN for node discovery - -The use of FQDN can resolve a situation where if IP addresses are re-used across different clusters, the nodes in one cluster could mistakenly discover nodes in another cluster. - -It can also be useful when using services (for example, Kubernetes) where the IP of the services is different from the actual node IP, but it proxies the connection to the node. - -#### Environment variables for FQDN node discovery - -`RAFT_ENABLE_FQDN_RESOLVER` is a Boolean flag. This flag enables the FQDN resolver. If set to `true`, Weaviate uses the FQDN resolver to resolve the node name to the node IP address. If set to `false`, Weaviate uses the memberlist lookup to resolve the node name to the node IP address. The default value is `false`. - -`RAFT_FQDN_RESOLVER_TLD` is a string that is appended in the format `[node-id].[tld]` when resolving a node-id to an IP address, where `[tld]` is the top-level domain. - -To use this feature, set `RAFT_ENABLE_FQDN_RESOLVER` to `true`. - -
- ## Metadata replication: Raft Weaviate uses the [Raft consensus algorithm](https://raft.github.io/) for metadata replication, implemented with Hashicorp's [raft library](https://pkg.go.dev/github.com/hashicorp/raft). Metadata in this context includes collection definition and shard/tenant states. diff --git a/docs/weaviate/concepts/replication-architecture/consistency.md b/docs/weaviate/concepts/replication-architecture/consistency.md index afc3addc0..7f28f9be0 100644 --- a/docs/weaviate/concepts/replication-architecture/consistency.md +++ b/docs/weaviate/concepts/replication-architecture/consistency.md @@ -90,7 +90,7 @@ Adding or changing data objects are **write** operations. Write operations are tunable starting with Weaviate v1.18, to `ONE`, `QUORUM` (default) or `ALL`. In v1.17, write operations are always set to `ALL` (highest consistency). ::: -The main reason for introducing configurable write consistency in v1.18 is because that is also when automatic repairs are introduced. A write will always be written to n (replication factor) nodes, regardless of the chosen consistency level. The coordinator node however waits for acknowledgments from `ONE`, `QUORUM` or `ALL` nodes before it returns. To guarantee that a write is applied everywhere without the availability of repairs on read requests, write consistency is set to `ALL` for now. Possible settings in v1.18+ are: +The main reason for introducing configurable write consistency in v1.18 is because that is also when automatic repairs are introduced. A write will always be written to n (replication factor) nodes, regardless of the chosen consistency level. The coordinator node however waits for acknowledgments from `ONE`, `QUORUM` or `ALL` nodes before it returns. To guarantee that a write is acknowledged everywhere before the request returns, without relying on read-time repairs, set write consistency to `ALL`. Possible settings in v1.18+ are: * **ONE** - a write must receive an acknowledgment from at least one replica node. This is the fastest (most available), but least consistent option. * **QUORUM** - a write must receive an acknowledgment from at least `QUORUM` replica nodes. `QUORUM` is calculated as _n / 2 + 1_, where _n_ is the number of replicas (replication factor). For example, using a replication factor of 6, the quorum is 4, which means the cluster can tolerate 2 replicas down. * **ALL** - a write must receive an acknowledgment from all replica nodes. This is the most consistent, but 'slowest' (least available) option. @@ -197,7 +197,7 @@ Repair-on-read works well with one or two isolated repairs. Async replication is Async replication supplements the repair-on-read mechanism. If a node becomes inconsistent between sync checks, the repair-on-read mechanism catches the problem at read time. -To activate async replication, set `asyncEnabled` to true in the [`replicationConfig` section of your collection definition](../../manage-collections/multi-node-setup.mdx#replication-settings). Visit the [How-to: Replication](/deploy/configuration/replication.md#async-replication-settings) page to learn more about the available async replication settings. +As of Weaviate `v1.38`, async replication is enabled by default for any collection with a replication factor greater than `1`, there is no per-collection flag to enable it. To turn it off cluster-wide, set the `ASYNC_REPLICATION_DISABLED` environment variable to `true`. Visit the [How-to: Replication](/deploy/configuration/replication.md#async-replication-settings) page to learn more about the available async replication settings. #### Memory and performance considerations for async replication @@ -262,9 +262,9 @@ A larger hash tree means less data for each leaf to hash, leading to faster comp - `Number of Leaves = 2^20 = 1,048,576` :::note Default settings -The default hash tree height is `16` for single-tenant collections and `10` for multi-tenant collections. These defaults balance memory consumption with replication performance. Similarly, the default `maxWorkers` is `3` for single-tenant collections and `30` for multi-tenant collections. +The default hash tree height is `16` for single-tenant collections and `10` for multi-tenant collections. These defaults balance memory consumption with replication performance. -As of `v1.36`, these parameters can be configured per-collection via the [`asyncConfig`](/weaviate/config-refs/collections#async-config) object in `replicationConfig`, overriding the cluster-wide environment variable defaults. +As of `v1.36`, these parameters can be configured per-collection via the [`asyncConfig`](/weaviate/config-refs/collections#async-config) object in `replicationConfig`. Worker concurrency is no longer a per-collection setting and as of `v1.38` the cluster shares a single async replication worker pool sized by [`ASYNC_REPLICATION_SCHEDULER_WORKERS`](/deploy/configuration/env-vars/index.md#async-replication). ::: ### Deletion resolution strategies diff --git a/docs/weaviate/concepts/replication-architecture/index.md b/docs/weaviate/concepts/replication-architecture/index.md index bcfb94731..9305dc2a3 100644 --- a/docs/weaviate/concepts/replication-architecture/index.md +++ b/docs/weaviate/concepts/replication-architecture/index.md @@ -163,12 +163,6 @@ Read more about how replication works in Weaviate in [Philosophy](./philosophy.m See [how to configure replication](/deploy/configuration/replication.md). You can enable replication in the collection definition. In queries, you can [specify the desired consistency level](../../search/basics.md#replication). -## Roadmap - -* Not scheduled yet - * Multi-Datacenter replication (you can upvote this feature [here](https://github.com/weaviate/weaviate/issues/2436)) - - ## Related pages - [Configuration: Replication](/deploy/configuration/replication.md) diff --git a/docs/weaviate/concepts/replication-architecture/motivation.md b/docs/weaviate/concepts/replication-architecture/motivation.md index 779b7218b..45dc42625 100644 --- a/docs/weaviate/concepts/replication-architecture/motivation.md +++ b/docs/weaviate/concepts/replication-architecture/motivation.md @@ -84,7 +84,7 @@ When users are located in different regional areas (e.g. Iceland and Australia a Multi-DC replication also comes with the additional benefit that data is redundant on more physical locations, which means that in the rare case of an entire datacenter going down, data can still be served from another location. :::note -Note, Regional Proximity depends on the Multi-Datacenter feature of replication, which you can [vote for here](https://github.com/weaviate/weaviate/issues/2436). +Regional Proximity is enabled by [running a Weaviate cluster across data centers](./multi-dc.md). ::: ## Questions and feedback diff --git a/docs/weaviate/concepts/replication-architecture/multi-dc.md b/docs/weaviate/concepts/replication-architecture/multi-dc.md index 1c1196a7c..53296785f 100644 --- a/docs/weaviate/concepts/replication-architecture/multi-dc.md +++ b/docs/weaviate/concepts/replication-architecture/multi-dc.md @@ -1,22 +1,62 @@ --- title: Multi-Data center sidebar_position: 5 -description: "Future multi-data center replication capabilities for distributed deployments across geographic regions." +description: "Run a Weaviate cluster across multiple data centers (WAN) for lower latency and cross-region redundancy." image: og/docs/concepts.jpg # tags: ['architecture'] --- -Multi-Data center (Multi-DC) replication enables you to have multiple copies of the data on multiple servers across more than one data center. This form of replication is not yet supported in v1.17 and v1.18. The current version of replication is designed to support Multi-DC later on. If you are interested in this feature, upvote [this GitHub issue](https://github.com/weaviate/weaviate/issues/2436). +Multi-Data center (Multi-DC) replication enables you to have multiple copies of the data on multiple servers across more than one data center. Weaviate supports running a single cluster across multiple data centers (since `v1.31`), so you can place nodes in different geographic regions for lower latency and keep your data redundant across locations. Multi-DC replication is beneficial if you have user groups spread over different geographical locations (e.g. Iceland and Australia). When you place nodes in different local regions of user groups, latency can be decreased. Multi-DC replication also comes with the additional benefit that data is redundant on more physical locations, which means that in the rare case of an entire data center going down, data can still be served from another location. -For now, you can work under the assumption that all replica nodes are in the same data center. The advantage of this is that network requests between nodes are cheap and fast. The downside is that if the entire data center fails, there is no redundancy. This will be solved with Multi-DC, [when implemented](https://github.com/weaviate/weaviate/issues/2436)! +If all replica nodes are in the same data center, network requests between nodes are cheap and fast, but the whole deployment is at risk if that data center goes down. Spreading a cluster's nodes across data centers removes this single point of failure, so your data remains available even if an entire data center becomes unreachable. + +To run a cluster across data centers, tune Weaviate's inter-node networking for the higher-latency, lower-reliability links between regions. See [Running a single cluster across data centers (WAN)](#running-a-single-cluster-across-data-centers-wan) below.

Replication multi-dc

+## Running a single cluster across data centers (WAN) + +Weaviate can operate a single cluster across data centers (a wide-area network, or WAN) by adjusting its inter-node communication for high-latency, lower-reliability links. + +### Enabling WAN mode + +Set the `CLUSTER_ADVERTISE_ADDR` environment variable to enable WAN mode. When it is set, Weaviate switches its internal [memberlist](https://github.com/hashicorp/memberlist) configuration to `DefaultWANConfig`, which increases timeouts and relaxes failure-detection thresholds so they are suitable for cross-data center communication. + +### Key environment variables + +| Variable | Description | Required | +| --- | --- | --- | +| `CLUSTER_ADVERTISE_ADDR` | The public IP address that other nodes should use to reach this node. Setting this enables WAN mode. Must be a valid IP address (hostnames are rejected). | Yes (for WAN) | +| `CLUSTER_ADVERTISE_PORT` | The port to advertise to other nodes. If not set, it defaults to `CLUSTER_GOSSIP_BIND_PORT`. If set, it must be between `1024` and `65535`. | No | +| `CLUSTER_BIND_ADDR` | The local address to bind to. Defaults to `0.0.0.0`. | No | +| `CLUSTER_GOSSIP_BIND_PORT` | The port used for gossip (memberlist) traffic. Defaults to `7946`. | No | +| `CLUSTER_DATA_BIND_PORT` | The port used for data traffic. Defaults to `CLUSTER_GOSSIP_BIND_PORT + 1`. | No | +| `CLUSTER_JOIN` | A comma-separated list of `host:port` addresses of existing cluster members to join. | Yes (for joining) | + +### Example configuration + +Below is an example environment configuration for a node intended to participate in a cross-data center cluster: + +```yaml +environment: + - CLUSTER_HOSTNAME=weaviate-0 + - CLUSTER_ADVERTISE_ADDR=203.0.113.10 # Public IP of this node + - CLUSTER_JOIN=203.0.113.20:7946 # Public IP:Port of a node in another DC + - CLUSTER_BIND_ADDR=0.0.0.0 # Optional; this is already the default bind address + - CLUSTER_GOSSIP_BIND_PORT=7946 + - CLUSTER_DATA_BIND_PORT=7947 + - RAFT_BOOTSTRAP_EXPECT=3 # Use an odd number > 1 across DCs for faster, more reliable elections + - RAFT_JOIN=203.0.113.20:8300 # Join an existing Raft voter (default Raft port is 8300) +``` + +### Notes +- **Latency**: Cross-data center operations inherently have higher latency. Make sure your application timeouts account for this. +- **Security**: Gossip and data traffic are not encrypted by default. For cross-data center communication over the public internet, use a VPN or other secure tunnel, or make sure `CLUSTER_ADVERTISE_ADDR` is only reachable over a private network (for example, VPC peering). ## Questions and feedback diff --git a/docs/weaviate/config-refs/collections.mdx b/docs/weaviate/config-refs/collections.mdx index dad5f2226..1c233a1c7 100644 --- a/docs/weaviate/config-refs/collections.mdx +++ b/docs/weaviate/config-refs/collections.mdx @@ -677,10 +677,13 @@ More details about the `vectorIndexType` and `vectorIndexConfig` parameters can | Parameter | Type | Description | Default | Mutable | | :----------------- | :------ | :-------------------------------------------------------------------------------------------------------------------------------------------- | :------------------------ | :-------------------------------------- | | `factor` | Integer | The number of copies (replicas) to maintain for each shard. A factor of `3` means one primary and two replicas. | `1` | No | -| `asyncEnabled` | Boolean | Enable asynchronous replication. | `false` | Yes | | `deletionStrategy` | String | Strategy for handling deletions in replication. Can be `NoAutomatedResolution`, `DeleteOnConflict` or `TimeBasedResolution`. | `"TimeBasedResolution"` | Yes | | `asyncConfig` | Object | Configuration for async replication tuning. See [`asyncConfig` parameters](#async-config) below. Added in `v1.36` | See below | Yes | +:::note Async replication is on by default (`v1.38`) +The `asyncEnabled` flag has been removed. As of Weaviate `v1.38`, async replication runs automatically for any collection with a `factor` greater than `1`. To turn it off, set the [`ASYNC_REPLICATION_DISABLED`](/deploy/configuration/env-vars/index.md#async-replication) environment variable to `true`. +::: +
Example replication configuration - JSON object @@ -693,10 +696,8 @@ An example of a complete `replicationConfig` object: // highlight-start "replicationConfig": { "factor": 3, - "asyncEnabled": true, "deletionStrategy": "TimeBasedResolution", "asyncConfig": { - "maxWorkers": 5, "hashtreeHeight": 16, "frequency": 30000 } @@ -719,11 +720,9 @@ Some `asyncConfig` parameters have different defaults depending on whether the c | Parameter | Type | Description | Default (single-tenant) | Default (multi-tenant) | | :--- | :--- | :--- | :--- | :--- | -| `maxWorkers` | Integer | Maximum number of concurrent async replication workers for this collection. | `3` | `30` | | `hashtreeHeight` | Integer | Height of the hash tree used for data comparison between nodes. Min: `0`, Max: `20` | `16` | `10` | | `frequency` | Integer | Frequency of periodic data comparison between nodes, in milliseconds. | `30000` | `30000` | | `frequencyWhilePropagating` | Integer | Frequency of data comparison while propagation is active, in milliseconds. | `3000` | `3000` | -| `aliveNodesCheckingFrequency` | Integer | Frequency of checking node availability, in milliseconds. | `5000` | `5000` | | `loggingFrequency` | Integer | How often the async replication process logs its activity, in seconds. | `60` | `60` | | `diffBatchSize` | Integer | Number of object keys fetched per request during comparison. Min: `1`, Max: `10000` | `1000` | `1000` | | `diffPerNodeTimeout` | Integer | Timeout for a comparison response from a remote node, in seconds. | `10` | `10` | diff --git a/docs/weaviate/config-refs/datatypes.md b/docs/weaviate/config-refs/datatypes.md index 286a3c4b5..eafcf0828 100644 --- a/docs/weaviate/config-refs/datatypes.md +++ b/docs/weaviate/config-refs/datatypes.md @@ -534,10 +534,10 @@ The `object` type allows you to store nested data as a JSON object that can be n For example, a `Person` collection could have an `address` property as an object. It could in turn include nested properties such as `street` and `city`: -:::note Limitations -Currently, `object` and `object[]` datatype properties are not indexed and not vectorized. +:::note Indexing and filtering + +`object` and `object[]` properties are not vectorized — only their leaf scalars are stored in the inverted index. From Weaviate `v1.38` (preview), you can filter on nested-object leaves using a dotted path syntax. See [Filter on nested object properties](../search/filters.md#filter-on-nested-object-properties). -Future plans include the ability to index nested properties, for example to allow for filtering on nested properties and vectorization options. ::: ### Examples diff --git a/docs/weaviate/configuration/mcp-server.mdx b/docs/weaviate/configuration/mcp-server.mdx index 18a83ffe4..d3b5bba9b 100644 --- a/docs/weaviate/configuration/mcp-server.mdx +++ b/docs/weaviate/configuration/mcp-server.mdx @@ -15,6 +15,7 @@ faq: - question: How do I request a new MCP tool or feature? answer: >- Open a feature request on the Weaviate GitHub repo at https://github.com/weaviate/weaviate/issues/new/choose. Pick the "Feature request" template and describe the tool, parameter, or capability you'd like the MCP server to expose, with a concrete use case. + # tags: ["mcp", "configuration"] --- @@ -155,11 +156,11 @@ The MCP server is built into Weaviate but is **disabled by default** for securit To enable and configure the server, set the following [environment variables](/docs/deploy/configuration/env-vars/index.md) in your Weaviate configuration (e.g., `docker-compose.yml`): -| Environment Variable | Default | Description | -| --------------------------------------------------------------------------------------------------- | ------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [`MCP_SERVER_ENABLED`](/deploy/configuration/env-vars#MCP_SERVER_ENABLED) | `false` | **Required.** Set to `true` to start the MCP server. | -| [`MCP_SERVER_WRITE_ACCESS_ENABLED`](/deploy/configuration/env-vars#MCP_SERVER_WRITE_ACCESS_ENABLED) | `false` | When `true`, enables write tools (`weaviate-objects-upsert`). Default is read-only. | -| [`MCP_SERVER_CONFIG_PATH`](/deploy/configuration/env-vars#MCP_SERVER_CONFIG_PATH) | `""` | Path to a YAML file for customizing tool descriptions (useful for prompt engineering the LLM's understanding of your specific data). If not provided or file malformed, the default descriptions from the [source code](https://github.com/weaviate/weaviate/tree/main/adapters/handlers/mcp) will be used. | +| Environment Variable | Default | Runtime-configurable | Description | +| --------------------------------------------------------------------------------------------------- | ------- | -------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [`MCP_SERVER_ENABLED`](/deploy/configuration/env-vars#MCP_SERVER_ENABLED) | `false` | from `v1.38` | **Required.** Set to `true` to start the MCP server. | +| [`MCP_SERVER_WRITE_ACCESS_ENABLED`](/deploy/configuration/env-vars#MCP_SERVER_WRITE_ACCESS_ENABLED) | `false` | from `v1.38` | When `true`, enables write tools (`weaviate-objects-upsert`). Default is read-only. | +| [`MCP_SERVER_CONFIG_PATH`](/deploy/configuration/env-vars#MCP_SERVER_CONFIG_PATH) | `""` | No | Path to a YAML file for customizing tool descriptions (useful for prompt engineering the LLM's understanding of your specific data). If not provided or file malformed, the default descriptions from the [source code](https://github.com/weaviate/weaviate/tree/main/adapters/handlers/mcp) will be used. Tool descriptions are baked into the tool schemas at registration, so this flag remains startup-only. | ### Permissions @@ -283,6 +284,14 @@ Batch inserts or updates objects. --- +## Monitoring + +From `v1.38`, the MCP server emits six Prometheus metrics under the `weaviate_mcp_*` prefix on the existing [Prometheus endpoint](/deploy/configuration/monitoring.md). Use them to track tool traffic, latency, auth failures, and the live state of the write-access flag. + +See [Monitoring → MCP server](/deploy/configuration/monitoring.md#mcp-server) for the full label catalogue and the rest of Weaviate's Prometheus surface. + +--- + ## Further resources - [Vibe coding - Best practices](../best-practices/code-generation.md) diff --git a/docs/weaviate/manage-collections/collection-operations.mdx b/docs/weaviate/manage-collections/collection-operations.mdx index e920e093c..5d61a4ecd 100644 --- a/docs/weaviate/manage-collections/collection-operations.mdx +++ b/docs/weaviate/manage-collections/collection-operations.mdx @@ -380,6 +380,8 @@ This configuration for nested objects defines the following: } ``` +To filter on values inside nested objects, see [Filter on nested object properties](../search/filters.md#filter-on-nested-object-properties). +
diff --git a/docs/weaviate/manage-collections/multi-node-setup.mdx b/docs/weaviate/manage-collections/multi-node-setup.mdx index 82160978a..051d3d7e3 100644 --- a/docs/weaviate/manage-collections/multi-node-setup.mdx +++ b/docs/weaviate/manage-collections/multi-node-setup.mdx @@ -85,10 +85,8 @@ curl \ ], "replicationConfig": { "factor": 3, - "asyncEnabled": true, "deletionStrategy": "TimeBasedResolution", "asyncConfig": { - "maxWorkers": 5, "hashtreeHeight": 16, "frequency": 30000 } diff --git a/docs/weaviate/search/bm25.md b/docs/weaviate/search/bm25.md index 456415bca..205527572 100644 --- a/docs/weaviate/search/bm25.md +++ b/docs/weaviate/search/bm25.md @@ -16,6 +16,7 @@ import GoCode from '!!raw-loader!/\_includes/code/howto/go/docs/mainpkg/search-b import JavaV6Code from "!!raw-loader!/\_includes/code/java-v6/src/test/java/SearchKeywordTest.java"; import CSharpCode from "!!raw-loader!/\_includes/code/csharp/SearchKeywordTest.cs"; import GQLCode from '!!raw-loader!/\_includes/code/howto/search.bm25.gql.py'; +import BoostPreview from '/_includes/feature-notes/boost.mdx'; `Keyword` search, also called "BM25 (Best match 25)" or "sparse vector" search, returns objects that have the highest BM25F scores. @@ -764,6 +765,14 @@ Set the tokenization method to `trigram` at the property level when creating you ::: +## Soft-rank with Boost + + + +Keyword (BM25) queries accept an optional `boost` argument that promotes or demotes matching documents without removing them — useful for biasing results by recency, popularity, a soft filter, or another property. Matching documents move up. Everything else stays in the results but ranks lower. + +See [Boost](./boost.md) for the supported condition types (filter, property value, time decay, numeric decay), curve choices, blending semantics, and depth tuning. + ## Further resources - [Connect to Weaviate](../connections/index.mdx) diff --git a/docs/weaviate/search/boost.md b/docs/weaviate/search/boost.md new file mode 100644 index 000000000..7bc9ecd75 --- /dev/null +++ b/docs/weaviate/search/boost.md @@ -0,0 +1,196 @@ +--- +title: Boost +sidebar_position: 76 +image: og/docs/howto.jpg +description: Soft-rank vector, hybrid, and BM25 results — promote or demote matching documents without filtering them out. Worked Python examples for filter, property, time-decay, numeric-decay, and blended boosts. +# tags: ['boost', 'search', 'ranking'] +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock'; +import PyCode from '!!raw-loader!/_includes/code/howto/search.boost.py'; +import BoostPreview from '/_includes/feature-notes/boost.mdx'; + + + +**Boost** soft-ranks search results — promote or demote matching documents without removing them from the result set. Matching documents move up. Non-matching documents stay in the results but rank lower. + +Apply boost to vector, hybrid, BM25, near-text, near-vector, near-object, and aggregate queries. + +## How it works + +A boost is a **post-retrieval rescorer**: + +1. The primary search (vector, hybrid, BM25, ...) fetches `depth` candidate results. Set `depth` higher than `offset + limit` if you want boost to consider candidates beyond the first page. +2. The boost scorer rescores those candidates in memory by evaluating each condition per candidate, normalizing per result set, and blending with the primary score. There are no new index queries — the cost is per-candidate in-memory scoring, not extra shard fan-out. Both primary and boost scores are min-max normalized into `[0, 1]` before blending, and the final score is renormalized to `[0, 1]`. +3. The user's original `offset` and `limit` are applied **after** the re-sort. + +## Condition types + +A boost is one or more **conditions**, blended into a single rescore. Every condition is one of: filter, property value, time decay, or numeric decay. + +### Filter condition (soft `WHERE`) + +Score is `1` if the result matches the filter, `0` if not. Non-matching documents stay in the result set but rank lower than matching ones. Supported filter operators: `Equal`, `NotEqual`, `GreaterThan`, `GreaterThanEqual`, `LessThan`, `LessThanEqual`, `And`, `Or`, `Not`. (`Like`, `IsNull`, geo operators, and ref-path filters are not supported in boost conditions.) + + + + + + + +### Property-value condition + +Continuous score proportional to a numeric property's value (`likes`, `downloads`, `popularity`, ...). The raw value is optionally modified, then min-max normalized to `[0, 1]` across the result set. + +The `name` argument is required, only numeric properties (`int`, `number`) are supported. + +| Modifier | Effect | When to use | +|---|---|---| +| `NONE` (default) | `score = value` | Values in a narrow range. | +| `LOG1P` | `score = log(1 + value)` | Long-tail dampening (e.g. download counts from `5` to `5_000_000`). | +| `SQRT` | `score = sqrt(value)` | Milder long-tail dampening. | + + + + + + + +For "closer to a specific value is better" instead of "higher is better", use [numeric decay](#numeric-decay) below. + +### Time decay + +Continuous `[0, 1]` score that **decays with distance from an origin time**. The canonical use case is "boost more recent documents". + +| Parameter | Required | Notes | +|---|---|---| +| `property` | Yes | Name of a `date` property. | +| `origin` | No | `"now"` (default), a `datetime`, or an ISO string. | +| `scale` | Yes | Distance at which the score equals `decay`. Accepts a `timedelta` or duration string (`"7d"`, `"6h"`, `"30m"`). | +| `offset` | No | Distance below which the score is exactly `1`. Default `0`. | +| `curve` | No | `EXPONENTIAL` (default), `GAUSSIAN`, or `LINEAR`. See [Curves](#curves) below. | +| `decay` | No | Score at `scale` distance. Default `0.5`. Range `(0, 1]`. | + + + + + + + +### Numeric decay {#numeric-decay} + +Like time decay but for numeric (`int`, `number`) properties. Use this when "closer to X is better" — prices near a target, distances near a coordinate, ages near a band. Same `scale` / `offset` / `decay` / `curve` semantics as time decay, with all values expressed as numbers. + +`scale` must be `> 0` and `decay` (if set) must be in `(0, 1]` — same as time decay. + + + + + + + +## Curves + +The three decay curves shape how score falls off with distance. At `distance == 0` the score is always `1`. At `distance == scale` the score is always exactly `decay`. Past `scale` they behave differently: + +| Curve | Shape | When to use | +|---|---|---| +| `EXPONENTIAL` (default) | Heavy tail — score halves geometrically every `scale` past the origin. | "Recency matters, but don't aggressively flatten older items to zero." | +| `GAUSSIAN` | Bell curve — sharp falloff past `scale`. | "Items close to the origin are great, items far away are nearly worthless." | +| `LINEAR` | Straight line — score reaches zero at a finite distance past `scale`. | "Predictable falloff with a clear cutoff." | + +Only these three values are accepted — anything else is rejected at request time. + +## Blending and weights + +A boost must carry **at least one** and **at most 20** conditions. Use `Boost.blend(...)` to combine multiple conditions into one rescore. Each condition can carry its own `weight` (default `1.0`). The outer `weight` (default `0.5`) controls how much the combined boost affects the final score. + +``` +final_score = (1 − weight) · primary_norm + weight · boost_norm +``` + +- **`weight`** — the outer blending weight, in `[0, 1]`. Defaults to `0.5`. +- **`weight: 0`** is a no-op: the boost short-circuits and primary results are returned unchanged. +- **Per-condition `weight`** — a `float` defaulting to `1.0`. Use it to balance multiple boosts ("recency twice as important as popularity"). +- **Negative per-condition `weight`** — *demotes* matching documents. They stay in the result set but rank lower than non-matching ones. + + + + + + + +### Negative weights demote + +A condition with `weight: -1.0` (or `-2.0`, etc.) reverses the effect: documents that match the condition rank below non-matching ones instead of above them. They are not removed. This is useful for deprioritizing — for example, surfacing drafts last without filtering them out. + + + + + + + +## Depth and pagination + +`depth` controls the **candidate pool**. The primary search fetches `depth` results before the boost rescorer runs. After rescoring, the user's `offset` and `limit` are applied. + +| Property | Value | +|---|---| +| Default | `100` | +| Operator override | [`QUERY_BOOST_DEFAULT_DEPTH`](/deploy/configuration/env-vars#QUERY_BOOST_DEFAULT_DEPTH) env var | +| Hard cap | `QUERY_MAXIMUM_RESULTS` (cluster-wide limit) | +| Lower bound | At least `offset + limit` — boost always sees enough to fill the page | +| Accepted range | `≥ 0` — `0` means "use the default" | + +:::tip Raise `depth` to reorder beyond the top page + +Boost can only reorder what the primary search already retrieved. If your boost should reorder past the default top-100 — for example, to pull a popular older article back to page 1 — pass a higher `depth` on the query. + +Higher `depth` increases the primary search cost (BM25 / vector index work) and per-candidate scoring cost. Set it as tight as your use case allows. + +::: + +## Further resources + +- [Rerank](./rerank.md) — second-stage reranking with an external model. +- [Hybrid search](./hybrid.md) — the BM25 / vector `alpha` blend. +- [Filters](./filters.md) — hard filters (remove non-matching docs). +- [BM25](./bm25.md) — keyword search. + +## Questions and feedback + +import DocsFeedback from '/\_includes/docs-feedback.mdx'; + + diff --git a/docs/weaviate/search/filters.md b/docs/weaviate/search/filters.md index 6e6451132..29ba28f5e 100644 --- a/docs/weaviate/search/filters.md +++ b/docs/weaviate/search/filters.md @@ -10,6 +10,7 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import FilteredTextBlock from '@site/src/components/Documentation/FilteredTextBlock'; import PyCode from '!!raw-loader!/\_includes/code/howto/search.filters.py'; +import PyCodeNested from '!!raw-loader!/\_includes/code/howto/search.filters.nested.py'; import PyCodeV3 from '!!raw-loader!/\_includes/code/howto/search.filters-v3.py'; import JavaScriptCode from '!!raw-loader!/\_includes/code/howto/search.filters.ts'; import GoCode from '!!raw-loader!/\_includes/code/howto/go/docs/mainpkg/search-filters_test.go'; @@ -196,9 +197,9 @@ The output is like this:
-## Nested filters +## Combine filters with `And` or `Or` -You can group and nest filters. +Group and nest filter conditions with `And` and `Or` operators to express compound logic. @@ -1064,6 +1065,123 @@ This filter requires the [property null state](../config-refs/indexing/inverted- +## Filter on nested object properties + +:::caution Preview feature + +Available from Weaviate `v1.38` as a preview, gated by the `WEAVIATE_PREVIEW_NESTED_FILTERING=on` environment variable on the server. The path syntax and operator semantics are stable, but the on-disk encoding may change before GA — don't rely on persistent state from preview clusters carrying over to the GA release. The env var is removed at GA and the feature is enabled unconditionally. + +::: + +[`object` and `object[]` properties](../config-refs/datatypes.md#object) carry their own nested schemas. To filter on a value inside a nested object, use a single dotted path naming the path from the parent property down to the leaf you want to compare. + +Given a collection like this: + + + +The filter property is a single dotted path. The dot is the only separator. An optional `[N]` after any segment pins that segment to an array index (0-based). + +| Path | Meaning | +|---|---| +| `cars.make` | Any car's `make` (matches if **any** element of the `cars` array has it) | +| `cars[0].make` | The first car's `make` (positional) | +| `cars.tires.width` | Any tire on any car (recursive across two `object[]` levels) | +| `cars[1].tires[2].brand` | The second car's third tire's `brand` (positional through nesting) | + +`[N]` on a segment requires that segment to be an `object[]` (array). Every intermediate segment must be `object` or `object[]` — you cannot pivot through a scalar. The leaf may be any supported scalar type. + +### Match any element (default) + +A path without `[N]` markers matches if **any** element in the parent array satisfies the condition. + + + + + + + +### Match by position + +Use `[N]` to pin a path segment to a specific array index. Indices are 0-based. + + + + + + + +### Same-element correlation across leaves + +Combining two leaf filters with `And` matches when **the same element** in the parent array satisfies both. A document with one car `(Toyota, blue)` and another `(Honda, red)` would not match `cars.make = "Toyota" AND cars.color = "red"` — both conditions must hold on the **same** car. + + + + + + + +### Deep / recursive paths + +`object[]` can nest inside `object[]` to any depth. Each segment in the dotted path traverses one level. + + + + + + + +### Check whether a nested object is absent + +Pointing a path at an `object` or `object[]` segment (rather than a scalar leaf) is only valid with `IsNull`, which asks whether that whole sub-object is present. + + + + + + + +### Limitations + +:::note + +- **Allowed leaf data types**: `text`, `int`, `number`, `boolean`, `date`, `uuid`, and their array variants. `blob`, `blobHash`, `geoCoordinates`, `phoneNumber`, and cross-references (`cref`) are not allowed inside nested objects for nested filtering. +- **`IndexFilterable` is required**: nested filtering uses the filterable inverted index on each leaf. `IndexRangeFilters` and `IndexSearchable` flags exist on nested-property definitions but are not yet exercised by the nested searcher — range filters on nested numeric leaves currently use the filterable bucket. +- **Tokenization matters**: nested `text` leaves use the same tokenization options as flat properties. For exact-match filters on names, codes, or identifiers, set `tokenization: field` on the leaf so the value is stored as a single token. +- **Reference-path vs nested-path**: a reference-path filter is a multi-element `Path` (`["inCity", "City", "name"]`) traversing cross-references; a nested-path filter is a **single-element** path with dots inside it (`["cars.make"]`). + +::: + ## Filter considerations ### Tokenization diff --git a/docs/weaviate/search/hybrid.md b/docs/weaviate/search/hybrid.md index 6f9695763..180ad69c6 100644 --- a/docs/weaviate/search/hybrid.md +++ b/docs/weaviate/search/hybrid.md @@ -16,6 +16,7 @@ import GoCode from '!!raw-loader!/\_includes/code/howto/go/docs/mainpkg/search-h import JavaV6Code from "!!raw-loader!/\_includes/code/java-v6/src/test/java/SearchHybridTest.java"; import CSharpCode from "!!raw-loader!/\_includes/code/csharp/SearchHybridTest.cs"; import GQLCode from '!!raw-loader!/\_includes/code/howto/search.hybrid.gql.py'; +import BoostPreview from '/_includes/feature-notes/boost.mdx'; `Hybrid` search combines the results of a vector search and a keyword (BM25F) search by fusing the two result sets. @@ -1038,6 +1039,16 @@ import TokenizationNote from '/\_includes/tokenization.mdx' +## Soft-rank with Boost + + + +Hybrid queries accept an optional `boost` argument that promotes or demotes matching documents without removing them — useful for biasing results by recency, popularity, a soft filter, or another property. + +The boost runs once over the **fused** hybrid result. The BM25 and vector sub-search legs do not see the boost themselves. Hybrid's own `alpha` blend runs first, and the boost rescores the fused candidate pool on top. + +See [Boost](./boost.md) for the supported condition types (filter, property value, time decay, numeric decay), curve choices, blending semantics, and depth tuning. + ## Related pages - [Connect to Weaviate](/weaviate/connections/index.mdx) diff --git a/docs/weaviate/search/rerank.md b/docs/weaviate/search/rerank.md index 7029d25f7..2dac8f571 100644 --- a/docs/weaviate/search/rerank.md +++ b/docs/weaviate/search/rerank.md @@ -16,6 +16,7 @@ import SimilarityPyCode from '!!raw-loader!/_includes/code/howto/search.similari import SimilarityPyCodeV3 from '!!raw-loader!/_includes/code/howto/search.similarity-v3.py'; import SimilarityTSCode from '!!raw-loader!/_includes/code/howto/search.similarity.ts'; import GoCode from '!!raw-loader!/_includes/code/howto/go/docs/mainpkg/search-rerank_test.go'; +import BoostPreview from '/_includes/feature-notes/boost.mdx'; Reranking modules reorder the search result set according to a different set of criteria or a different (e.g. more expensive) algorithm. @@ -196,6 +197,12 @@ The response should look like this: +## Soft-rank with Boost + + + +For lightweight result reordering based on filters, property values, or time / numeric decay — without calling an external rerank model — use [Boost](./boost.md). Rerank and Boost can be used independently. Pick rerank when you need a smarter model to re-rank the top-N, and Boost when you want to bias by simple signals already on the objects. + ## Related pages - [Connect to Weaviate](/weaviate/connections/index.mdx) diff --git a/docs/weaviate/search/similarity.md b/docs/weaviate/search/similarity.md index fced2f92c..5a4c2fe95 100644 --- a/docs/weaviate/search/similarity.md +++ b/docs/weaviate/search/similarity.md @@ -15,6 +15,7 @@ import TSCode from '!!raw-loader!/\_includes/code/howto/search.similarity.ts'; import GoCode from '!!raw-loader!/\_includes/code/howto/go/docs/mainpkg/search-similarity_test.go'; import JavaV6Code from "!!raw-loader!/\_includes/code/java-v6/src/test/java/SearchSimilarityTest.java"; import CSharpCode from "!!raw-loader!/\_includes/code/csharp/SearchSimilarityTest.cs"; +import BoostPreview from '/_includes/feature-notes/boost.mdx'; Vector search returns the objects with most similar vectors to that of the query. @@ -716,6 +717,14 @@ Important notes: A larger candidate set (higher top-level `limit`) gives MMR more results to choose from, improving diversity at the cost of slightly more computation. A good starting point is setting the candidate `limit` to 2–4x the MMR `limit`. ::: +## Soft-rank with Boost + + + +Vector search queries accept an optional `boost` argument that promotes or demotes matching documents without removing them — useful for biasing results by recency, popularity, a soft filter, or another property. Matching documents move up. Everything else stays in the results but ranks lower. + +See [Boost](./boost.md) for the supported condition types (filter, property value, time decay, numeric decay), curve choices, blending semantics, and depth tuning. + ## Related pages - [Connect to Weaviate](/weaviate/connections/index.mdx) diff --git a/docusaurus.config.js b/docusaurus.config.js index 5bf32c414..f5242667c 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -54,7 +54,7 @@ const config = { cdn: "https://cdn.jsdelivr.net/npm/@scalar/api-reference@1.49.0", configuration: { spec: { - url: "https://raw.githubusercontent.com/weaviate/weaviate/openapi-for-docs/openapi-specs/schema.json", + url: "https://raw.githubusercontent.com/weaviate/weaviate/v1-38/openapi-for-docs/openapi-specs/schema.json", }, hideModels: true, // showSidebar: true, diff --git a/sidebars.js b/sidebars.js index 07aecc234..f092ace48 100644 --- a/sidebars.js +++ b/sidebars.js @@ -654,6 +654,7 @@ const sidebars = { "weaviate/search/multi-vector", "weaviate/search/generative", "weaviate/search/rerank", + "weaviate/search/boost", "weaviate/search/aggregate", "weaviate/search/filters", "weaviate/search/query-profile", diff --git a/tests/docker-compose-anon-2.yml b/tests/docker-compose-anon-2.yml index 90497663d..f659f0736 100644 --- a/tests/docker-compose-anon-2.yml +++ b/tests/docker-compose-anon-2.yml @@ -8,7 +8,7 @@ services: - '8080' - --scheme - http - image: cr.weaviate.io/semitechnologies/weaviate:1.37.2 + image: cr.weaviate.io/semitechnologies/weaviate:1.38.0-rc.1 ports: - 8090:8080 - 50061:50051 diff --git a/tests/docker-compose-anon-bind.yml b/tests/docker-compose-anon-bind.yml index f821a5564..8e665f13f 100644 --- a/tests/docker-compose-anon-bind.yml +++ b/tests/docker-compose-anon-bind.yml @@ -8,7 +8,7 @@ services: - '8080' - --scheme - http - image: cr.weaviate.io/semitechnologies/weaviate:1.37.2 + image: cr.weaviate.io/semitechnologies/weaviate:1.38.0-rc.1 ports: - 8380:8080 - 50351:50051 diff --git a/tests/docker-compose-anon-clip.yml b/tests/docker-compose-anon-clip.yml index 861f96773..57bd44df6 100644 --- a/tests/docker-compose-anon-clip.yml +++ b/tests/docker-compose-anon-clip.yml @@ -8,7 +8,7 @@ services: - '8080' - --scheme - http - image: cr.weaviate.io/semitechnologies/weaviate:1.37.2 + image: cr.weaviate.io/semitechnologies/weaviate:1.38.0-rc.1 ports: - 8280:8080 - 50251:50051 diff --git a/tests/docker-compose-anon-offload.yml b/tests/docker-compose-anon-offload.yml index 2c1639680..7752e72dd 100644 --- a/tests/docker-compose-anon-offload.yml +++ b/tests/docker-compose-anon-offload.yml @@ -8,7 +8,7 @@ services: - '8080' - --scheme - http - image: cr.weaviate.io/semitechnologies/weaviate:1.37.2 + image: cr.weaviate.io/semitechnologies/weaviate:1.38.0-rc.1 ports: - 8080:8080 - 50051:50051 diff --git a/tests/docker-compose-anon.yml b/tests/docker-compose-anon.yml index bdfa367e2..d56a3f729 100644 --- a/tests/docker-compose-anon.yml +++ b/tests/docker-compose-anon.yml @@ -8,7 +8,7 @@ services: - '8080' - --scheme - http - image: cr.weaviate.io/semitechnologies/weaviate:1.37.2 + image: cr.weaviate.io/semitechnologies/weaviate:1.38.0-rc.1 ports: - 8080:8080 - 50051:50051 @@ -28,6 +28,7 @@ services: OBJECTS_TTL_DELETE_SCHEDULE: '*/10 * * * * *' EXPORT_ENABLED: 'true' EXPORT_DEFAULT_PATH: '/var/lib/weaviate/exports' + WEAVIATE_PREVIEW_NESTED_FILTERING: 'on' text2vec-transformers: image: cr.weaviate.io/semitechnologies/transformers-inference:sentence-transformers-multi-qa-MiniLM-L6-cos-v1 ollama: diff --git a/tests/docker-compose-rbac.yml b/tests/docker-compose-rbac.yml index 049e7537b..ce5b8b069 100644 --- a/tests/docker-compose-rbac.yml +++ b/tests/docker-compose-rbac.yml @@ -7,7 +7,7 @@ services: - '8080' - --scheme - http - image: cr.weaviate.io/semitechnologies/weaviate:1.37.2 + image: cr.weaviate.io/semitechnologies/weaviate:1.38.0-rc.1 ports: - 8580:8080 - 50551:50051 diff --git a/tests/docker-compose-three-nodes.yml b/tests/docker-compose-three-nodes.yml index b83a21b0a..a3d0577d6 100644 --- a/tests/docker-compose-three-nodes.yml +++ b/tests/docker-compose-three-nodes.yml @@ -8,7 +8,7 @@ services: - '8080' - --scheme - http - image: cr.weaviate.io/semitechnologies/weaviate:1.37.2 + image: cr.weaviate.io/semitechnologies/weaviate:1.38.0-rc.1 restart: on-failure:0 ports: - "8180:8080" @@ -36,7 +36,7 @@ services: - '8080' - --scheme - http - image: cr.weaviate.io/semitechnologies/weaviate:1.37.2 + image: cr.weaviate.io/semitechnologies/weaviate:1.38.0-rc.1 restart: on-failure:0 ports: - "8181:8080" @@ -65,7 +65,7 @@ services: - '8080' - --scheme - http - image: cr.weaviate.io/semitechnologies/weaviate:1.37.2 + image: cr.weaviate.io/semitechnologies/weaviate:1.38.0-rc.1 restart: on-failure:0 ports: - "8182:8080" diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index 93a95c0d7..ed6fbb308 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -8,7 +8,7 @@ services: - '8080' - --scheme - http - image: cr.weaviate.io/semitechnologies/weaviate:1.37.2 + image: cr.weaviate.io/semitechnologies/weaviate:1.38.0-rc.1 ports: - 8099:8080 - 50052:50051 diff --git a/tests/test_python.py b/tests/test_python.py index f412afc5a..6cc18242f 100644 --- a/tests/test_python.py +++ b/tests/test_python.py @@ -192,7 +192,13 @@ def test_modules(empty_weaviates, script_loc): "./_includes/code/howto/search.aggregate.py", "./_includes/code/howto/search.generative.py", "./_includes/code/howto/search.rerank.py", - "./_includes/code/howto/search.multi-target-v4.py" + "./_includes/code/howto/search.multi-target-v4.py", + # Nested-object filtering preview (v1.38). Requires WEAVIATE_PREVIEW_NESTED_FILTERING=on, + # which is set on the anon test instance (tests/docker-compose-anon.yml). + "./_includes/code/howto/search.filters.nested.py", + # Boost preview (v1.38). Re-enable once the weaviate-client release with Boost support + # (weaviate/weaviate-python-client#2030) is pinned in pyproject.toml — 4.21.0 lacks it. + # "./_includes/code/howto/search.boost.py", ], ) def test_search(empty_weaviates, script_loc): diff --git a/versions-config.json b/versions-config.json index 3afd0a68b..ea0898064 100644 --- a/versions-config.json +++ b/versions-config.json @@ -1,7 +1,7 @@ { "COMMENT1": "These values are used for yarn local yarn builds", "COMMENT2": "Build time values are set in _build_scripts/update-config-versions.js", - "weaviate_version": "1.37.2", + "weaviate_version": "1.38.0", "helm_version": "17.8.0", "weaviate_cli_version": "3.2.2", "weaviate_agents_version": "1.4.0",