mxgo-core/model.config.example.toml at master · mxgoai/mxgo-core · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
[[model]]
model_name = "gpt-4"

[model.litellm_params]
model = "azure/gpt-4"
base_url = "https://your-endpoint.openai.azure.com"
api_key = "your-key"
api_version = "2023-05-15"
weight = 5

[[model]]
model_name = "gpt-4"

[model.litellm_params]
model = "azure/gpt-4-1106-preview"
base_url = "https://your-endpoint-2.openai.azure.com"
api_key = "your-key-2"
api_version = "2023-05-15"
weight = 5

[[model]]
model_name = "gpt-4-reasoning"

[model.litellm_params]
model = "azure/gpt-4o-mini"
base_url = "https://your-endpoint-3.openai.azure.com"
api_key = "your-key-3"
api_version = "2023-05-15"
weight = 1

[[model]]
model_name = "ollama"

[model.litellm_params]
model = "ollama/tinyllama"
base_url = "http://localhost:11434"
api_key = "dummy"
api_version = ""
weight = 1

# Claude Sonnet via Amazon Bedrock
[[model]]
model_name = "claude-sonnet"

[model.litellm_params]
model = "bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0"
aws_access_key_id = "os.environ/AWS_ACCESS_KEY_ID"
aws_secret_access_key = "os.environ/AWS_SECRET_ACCESS_KEY"
aws_region_name = "os.environ/AWS_REGION_NAME"
weight = 3

# Claude Sonnet 3.5 via Amazon Bedrock (ARN-based inference profile)
[[model]]
model_name = "claude-sonnet"

[model.litellm_params]
model = "bedrock/arn:aws:bedrock:us-east-1:763126074217:inference-profile/us.anthropic.claude-3-5-sonnet-20241022-v2:0"
aws_access_key_id = "os.environ/AWS_ACCESS_KEY_ID"
aws_secret_access_key = "os.environ/AWS_SECRET_ACCESS_KEY"
aws_region_name = "os.environ/AWS_REGION_NAME"
weight = 5

# Alternative: Claude Sonnet using explicit inference profile parameter
[[model]]
model_name = "claude-sonnet"

[model.litellm_params]
model = "bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0"
aws_access_key_id = "os.environ/AWS_ACCESS_KEY_ID"
aws_secret_access_key = "os.environ/AWS_SECRET_ACCESS_KEY"
aws_region_name = "os.environ/AWS_REGION_NAME"
aws_bedrock_inference_profile = "arn:aws:bedrock:us-east-1:763126074217:inference-profile/us.anthropic.claude-3-5-sonnet-20241022-v2:0"
weight = 2

[router_config]
routing_strategy = "simple-shuffle"

[[router_config.fallbacks]]
gpt-4 = ["gpt-4-reasoning"]

# Add Claude Sonnet as fallback for GPT-4
[[router_config.fallbacks]]
claude-sonnet = ["gpt-4"]

[router_config.default_litellm_params]
drop_params = true