Skip to content

Commit 926ff5f

Browse files
api-clients-generation-pipeline[bot]ci.datadog-api-spec
api-clients-generation-pipeline[bot]
and
ci.datadog-api-spec
authored
Add OpenAPI documentation for cache property in dedupe processor in observability pipelines (#3173)
Co-authored-by: ci.datadog-api-spec <packages@datadoghq.com>
1 parent 96661fa commit 926ff5f

13 files changed

Lines changed: 363 additions & 0 deletions

‎.generator/schemas/v2/openapi.yaml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39514,6 +39514,8 @@ components:
3951439514

3951539515
**Supported pipeline types:** logs'
3951639516
properties:
39517+
cache:
39518+
$ref: '#/components/schemas/ObservabilityPipelineDedupeProcessorCache'
3951739519
display_name:
3951839520
$ref: '#/components/schemas/ObservabilityPipelineComponentDisplayName'
3951939521
enabled:
@@ -39551,6 +39553,19 @@ components:
3955139553
type: object
3955239554
x-pipeline-types:
3955339555
- logs
39556+
ObservabilityPipelineDedupeProcessorCache:
39557+
description: Configuration for the cache used to detect duplicates.
39558+
properties:
39559+
num_events:
39560+
description: The number of events to cache for duplicate detection.
39561+
example: 5000
39562+
format: int64
39563+
maximum: 1000000000
39564+
minimum: 1
39565+
type: integer
39566+
required:
39567+
- num_events
39568+
type: object
3955439569
ObservabilityPipelineDedupeProcessorMode:
3955539570
description: The deduplication mode to apply to the fields.
3955639571
enum:

‎docs/datadog_api_client.v2.model.rst‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17147,6 +17147,13 @@ datadog\_api\_client.v2.model.observability\_pipeline\_dedupe\_processor module
1714717147
:members:
1714817148
:show-inheritance:
1714917149

17150+
datadog\_api\_client.v2.model.observability\_pipeline\_dedupe\_processor\_cache module
17151+
--------------------------------------------------------------------------------------
17152+
17153+
.. automodule:: datadog_api_client.v2.model.observability_pipeline_dedupe_processor_cache
17154+
:members:
17155+
:show-inheritance:
17156+
1715017157
datadog\_api\_client.v2.model.observability\_pipeline\_dedupe\_processor\_mode module
1715117158
-------------------------------------------------------------------------------------
1715217159

‎examples/v2/observability-pipelines/CreatePipeline_3363445359.py‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
"""
2+
Create a pipeline with dedupe processor with cache returns "OK" response
3+
"""
4+
5+
from datadog_api_client import ApiClient, Configuration
6+
from datadog_api_client.v2.api.observability_pipelines_api import ObservabilityPipelinesApi
7+
from datadog_api_client.v2.model.observability_pipeline_config import ObservabilityPipelineConfig
8+
from datadog_api_client.v2.model.observability_pipeline_config_processor_group import (
9+
ObservabilityPipelineConfigProcessorGroup,
10+
)
11+
from datadog_api_client.v2.model.observability_pipeline_data_attributes import ObservabilityPipelineDataAttributes
12+
from datadog_api_client.v2.model.observability_pipeline_datadog_agent_source import (
13+
ObservabilityPipelineDatadogAgentSource,
14+
)
15+
from datadog_api_client.v2.model.observability_pipeline_datadog_agent_source_type import (
16+
ObservabilityPipelineDatadogAgentSourceType,
17+
)
18+
from datadog_api_client.v2.model.observability_pipeline_datadog_logs_destination import (
19+
ObservabilityPipelineDatadogLogsDestination,
20+
)
21+
from datadog_api_client.v2.model.observability_pipeline_datadog_logs_destination_type import (
22+
ObservabilityPipelineDatadogLogsDestinationType,
23+
)
24+
from datadog_api_client.v2.model.observability_pipeline_dedupe_processor import ObservabilityPipelineDedupeProcessor
25+
from datadog_api_client.v2.model.observability_pipeline_dedupe_processor_cache import (
26+
ObservabilityPipelineDedupeProcessorCache,
27+
)
28+
from datadog_api_client.v2.model.observability_pipeline_dedupe_processor_mode import (
29+
ObservabilityPipelineDedupeProcessorMode,
30+
)
31+
from datadog_api_client.v2.model.observability_pipeline_dedupe_processor_type import (
32+
ObservabilityPipelineDedupeProcessorType,
33+
)
34+
from datadog_api_client.v2.model.observability_pipeline_spec import ObservabilityPipelineSpec
35+
from datadog_api_client.v2.model.observability_pipeline_spec_data import ObservabilityPipelineSpecData
36+
37+
body = ObservabilityPipelineSpec(
38+
data=ObservabilityPipelineSpecData(
39+
attributes=ObservabilityPipelineDataAttributes(
40+
config=ObservabilityPipelineConfig(
41+
destinations=[
42+
ObservabilityPipelineDatadogLogsDestination(
43+
id="datadog-logs-destination",
44+
inputs=[
45+
"my-processor-group",
46+
],
47+
type=ObservabilityPipelineDatadogLogsDestinationType.DATADOG_LOGS,
48+
),
49+
],
50+
processor_groups=[
51+
ObservabilityPipelineConfigProcessorGroup(
52+
enabled=True,
53+
id="my-processor-group",
54+
include="service:my-service",
55+
inputs=[
56+
"datadog-agent-source",
57+
],
58+
processors=[
59+
ObservabilityPipelineDedupeProcessor(
60+
enabled=True,
61+
id="dedupe-processor",
62+
include="service:my-service",
63+
type=ObservabilityPipelineDedupeProcessorType.DEDUPE,
64+
fields=[
65+
"message",
66+
],
67+
mode=ObservabilityPipelineDedupeProcessorMode.MATCH,
68+
cache=ObservabilityPipelineDedupeProcessorCache(
69+
num_events=5000,
70+
),
71+
),
72+
],
73+
),
74+
],
75+
sources=[
76+
ObservabilityPipelineDatadogAgentSource(
77+
id="datadog-agent-source",
78+
type=ObservabilityPipelineDatadogAgentSourceType.DATADOG_AGENT,
79+
),
80+
],
81+
),
82+
name="Pipeline with Dedupe Cache",
83+
),
84+
type="pipelines",
85+
),
86+
)
87+
88+
configuration = Configuration()
89+
with ApiClient(configuration) as api_client:
90+
api_instance = ObservabilityPipelinesApi(api_client)
91+
response = api_instance.create_pipeline(body=body)
92+
93+
print(response)

‎examples/v2/observability-pipelines/CreatePipeline_581245895.py‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
"""
2+
Create a pipeline with dedupe processor without cache returns "OK" response
3+
"""
4+
5+
from datadog_api_client import ApiClient, Configuration
6+
from datadog_api_client.v2.api.observability_pipelines_api import ObservabilityPipelinesApi
7+
from datadog_api_client.v2.model.observability_pipeline_config import ObservabilityPipelineConfig
8+
from datadog_api_client.v2.model.observability_pipeline_config_processor_group import (
9+
ObservabilityPipelineConfigProcessorGroup,
10+
)
11+
from datadog_api_client.v2.model.observability_pipeline_data_attributes import ObservabilityPipelineDataAttributes
12+
from datadog_api_client.v2.model.observability_pipeline_datadog_agent_source import (
13+
ObservabilityPipelineDatadogAgentSource,
14+
)
15+
from datadog_api_client.v2.model.observability_pipeline_datadog_agent_source_type import (
16+
ObservabilityPipelineDatadogAgentSourceType,
17+
)
18+
from datadog_api_client.v2.model.observability_pipeline_datadog_logs_destination import (
19+
ObservabilityPipelineDatadogLogsDestination,
20+
)
21+
from datadog_api_client.v2.model.observability_pipeline_datadog_logs_destination_type import (
22+
ObservabilityPipelineDatadogLogsDestinationType,
23+
)
24+
from datadog_api_client.v2.model.observability_pipeline_dedupe_processor import ObservabilityPipelineDedupeProcessor
25+
from datadog_api_client.v2.model.observability_pipeline_dedupe_processor_mode import (
26+
ObservabilityPipelineDedupeProcessorMode,
27+
)
28+
from datadog_api_client.v2.model.observability_pipeline_dedupe_processor_type import (
29+
ObservabilityPipelineDedupeProcessorType,
30+
)
31+
from datadog_api_client.v2.model.observability_pipeline_spec import ObservabilityPipelineSpec
32+
from datadog_api_client.v2.model.observability_pipeline_spec_data import ObservabilityPipelineSpecData
33+
34+
body = ObservabilityPipelineSpec(
35+
data=ObservabilityPipelineSpecData(
36+
attributes=ObservabilityPipelineDataAttributes(
37+
config=ObservabilityPipelineConfig(
38+
destinations=[
39+
ObservabilityPipelineDatadogLogsDestination(
40+
id="datadog-logs-destination",
41+
inputs=[
42+
"my-processor-group",
43+
],
44+
type=ObservabilityPipelineDatadogLogsDestinationType.DATADOG_LOGS,
45+
),
46+
],
47+
processor_groups=[
48+
ObservabilityPipelineConfigProcessorGroup(
49+
enabled=True,
50+
id="my-processor-group",
51+
include="service:my-service",
52+
inputs=[
53+
"datadog-agent-source",
54+
],
55+
processors=[
56+
ObservabilityPipelineDedupeProcessor(
57+
enabled=True,
58+
id="dedupe-processor",
59+
include="service:my-service",
60+
type=ObservabilityPipelineDedupeProcessorType.DEDUPE,
61+
fields=[
62+
"message",
63+
],
64+
mode=ObservabilityPipelineDedupeProcessorMode.MATCH,
65+
),
66+
],
67+
),
68+
],
69+
sources=[
70+
ObservabilityPipelineDatadogAgentSource(
71+
id="datadog-agent-source",
72+
type=ObservabilityPipelineDatadogAgentSourceType.DATADOG_AGENT,
73+
),
74+
],
75+
),
76+
name="Pipeline with Dedupe No Cache",
77+
),
78+
type="pipelines",
79+
),
80+
)
81+
82+
configuration = Configuration()
83+
with ApiClient(configuration) as api_client:
84+
api_instance = ObservabilityPipelinesApi(api_client)
85+
response = api_instance.create_pipeline(body=body)
86+
87+
print(response)

‎src/datadog_api_client/v2/model/observability_pipeline_config_processor_item.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,9 @@ def __init__(self, **kwargs):
4848
:param mode: The processing mode.
4949
:type mode: ObservabilityPipelineDatadogTagsProcessorMode
5050
51+
:param cache: Configuration for the cache used to detect duplicates.
52+
:type cache: ObservabilityPipelineDedupeProcessorCache, optional
53+
5154
:param file: Defines a static enrichment table loaded from a CSV file.
5255
:type file: ObservabilityPipelineEnrichmentTableFile, optional
5356

‎src/datadog_api_client/v2/model/observability_pipeline_dedupe_processor.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@
1414

1515

1616
if TYPE_CHECKING:
17+
from datadog_api_client.v2.model.observability_pipeline_dedupe_processor_cache import (
18+
ObservabilityPipelineDedupeProcessorCache,
19+
)
1720
from datadog_api_client.v2.model.observability_pipeline_dedupe_processor_mode import (
1821
ObservabilityPipelineDedupeProcessorMode,
1922
)
@@ -25,6 +28,9 @@
2528
class ObservabilityPipelineDedupeProcessor(ModelNormal):
2629
@cached_property
2730
def openapi_types(_):
31+
from datadog_api_client.v2.model.observability_pipeline_dedupe_processor_cache import (
32+
ObservabilityPipelineDedupeProcessorCache,
33+
)
2834
from datadog_api_client.v2.model.observability_pipeline_dedupe_processor_mode import (
2935
ObservabilityPipelineDedupeProcessorMode,
3036
)
@@ -33,6 +39,7 @@ def openapi_types(_):
3339
)
3440

3541
return {
42+
"cache": (ObservabilityPipelineDedupeProcessorCache,),
3643
"display_name": (str,),
3744
"enabled": (bool,),
3845
"fields": ([str],),
@@ -43,6 +50,7 @@ def openapi_types(_):
4350
}
4451

4552
attribute_map = {
53+
"cache": "cache",
4654
"display_name": "display_name",
4755
"enabled": "enabled",
4856
"fields": "fields",
@@ -60,6 +68,7 @@ def __init__(
6068
include: str,
6169
mode: ObservabilityPipelineDedupeProcessorMode,
6270
type: ObservabilityPipelineDedupeProcessorType,
71+
cache: Union[ObservabilityPipelineDedupeProcessorCache, UnsetType] = unset,
6372
display_name: Union[str, UnsetType] = unset,
6473
**kwargs,
6574
):
@@ -68,6 +77,9 @@ def __init__(
6877
6978
**Supported pipeline types:** logs
7079
80+
:param cache: Configuration for the cache used to detect duplicates.
81+
:type cache: ObservabilityPipelineDedupeProcessorCache, optional
82+
7183
:param display_name: The display name for a component.
7284
:type display_name: str, optional
7385
@@ -89,6 +101,8 @@ def __init__(
89101
:param type: The processor type. The value should always be ``dedupe``.
90102
:type type: ObservabilityPipelineDedupeProcessorType
91103
"""
104+
if cache is not unset:
105+
kwargs["cache"] = cache
92106
if display_name is not unset:
93107
kwargs["display_name"] = display_name
94108
super().__init__(kwargs)

‎src/datadog_api_client/v2/model/observability_pipeline_dedupe_processor_cache.py‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License.
2+
# This product includes software developed at Datadog (https://www.datadoghq.com/).
3+
# Copyright 2019-Present Datadog, Inc.
4+
from __future__ import annotations
5+
6+
7+
from datadog_api_client.model_utils import (
8+
ModelNormal,
9+
cached_property,
10+
)
11+
12+
13+
class ObservabilityPipelineDedupeProcessorCache(ModelNormal):
14+
validations = {
15+
"num_events": {
16+
"inclusive_maximum": 1000000000,
17+
"inclusive_minimum": 1,
18+
},
19+
}
20+
21+
@cached_property
22+
def openapi_types(_):
23+
return {
24+
"num_events": (int,),
25+
}
26+
27+
attribute_map = {
28+
"num_events": "num_events",
29+
}
30+
31+
def __init__(self_, num_events: int, **kwargs):
32+
"""
33+
Configuration for the cache used to detect duplicates.
34+
35+
:param num_events: The number of events to cache for duplicate detection.
36+
:type num_events: int
37+
"""
38+
super().__init__(kwargs)
39+
40+
self_.num_events = num_events

‎src/datadog_api_client/v2/models/__init__.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3296,6 +3296,9 @@
32963296
)
32973297
from datadog_api_client.v2.model.observability_pipeline_decoding import ObservabilityPipelineDecoding
32983298
from datadog_api_client.v2.model.observability_pipeline_dedupe_processor import ObservabilityPipelineDedupeProcessor
3299+
from datadog_api_client.v2.model.observability_pipeline_dedupe_processor_cache import (
3300+
ObservabilityPipelineDedupeProcessorCache,
3301+
)
32993302
from datadog_api_client.v2.model.observability_pipeline_dedupe_processor_mode import (
33003303
ObservabilityPipelineDedupeProcessorMode,
33013304
)
@@ -8464,6 +8467,7 @@
84648467
"ObservabilityPipelineDatadogTagsProcessorType",
84658468
"ObservabilityPipelineDecoding",
84668469
"ObservabilityPipelineDedupeProcessor",
8470+
"ObservabilityPipelineDedupeProcessorCache",
84678471
"ObservabilityPipelineDedupeProcessorMode",
84688472
"ObservabilityPipelineDedupeProcessorType",
84698473
"ObservabilityPipelineDiskBufferOptions",

‎tests/v2/cassettes/test_scenarios/test_create_a_pipeline_with_dedupe_processor_with_cache_returns_ok_response.frozen‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
2026-02-09T09:53:31.840Z

0 commit comments

Comments
 (0)