Skip to content

chore: deprecate Claude 3.5 in the codebase #2472

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
sycai wants to merge 3 commits into
base: main
Choose a base branch
from
+72 −132
Open

chore: deprecate Claude 3.5 in the codebase #2472

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
20d8a52
chore: deprecated Claude 3.5 sonnet for TextGenerator
sycai Feb 20, 2026
023ad4b
update notebook too
sycai Feb 20, 2026
09e52d9
remove py env check
sycai Feb 20, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion bigframes/ml/llm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -877,7 +877,7 @@ class Claude3TextGenerator(base.RetriableRemotePredictor):
The model for natural language tasks. Possible values are "claude-3-sonnet", "claude-3-haiku", "claude-3-5-sonnet" and "claude-3-opus".
"claude-3-sonnet" (deprecated) is Anthropic's dependable combination of skills and speed. It is engineered to be dependable for scaled AI deployments across a variety of use cases.
"claude-3-haiku" is Anthropic's fastest, most compact vision and text model for near-instant responses to simple queries, meant for seamless AI experiences mimicking human interactions.
"claude-3-5-sonnet" is Anthropic's most powerful AI model and maintains the speed and cost of Claude 3 Sonnet, which is a mid-tier model.
"claude-3-5-sonnet" (deprecated) is Anthropic's most powerful AI model and maintains the speed and cost of Claude 3 Sonnet, which is a mid-tier model.
"claude-3-opus" (deprecated) is Anthropic's second-most powerful AI model, with strong performance on highly complex tasks.
https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/use-claude#available-claude-models
If no setting is provided, "claude-3-sonnet" will be used by default
Expand Down
158 changes: 62 additions & 96 deletions notebooks/remote_functions/remote_function_vertex_claude_model.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,20 +28,6 @@
"</table>"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import sys\n",
"\n",
"# Python 3.13 is not yet a supported runtime for remote functions.\n",
"# See: https://cloud.google.com/functions/docs/runtime-support#python for the supported runtimes.\n",
"if sys.version_info >= (3, 13, 0):\n",
" sys.exit(0)"
]
},
{
"cell_type": "markdown",
"metadata": {},
Expand Down Expand Up @@ -155,30 +141,6 @@
"execution_count": 4,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"Query job c4c27713-51c8-4293-8454-5c904df79318 is DONE. 0 Bytes processed. <a target=\"_blank\" href=\"https://console.cloud.google.com/bigquery?project=bigframes-dev&j=bq:us-east5:c4c27713-51c8-4293-8454-5c904df79318&page=queryresults\">Open Job</a>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"Query job 0b1b71d8-8546-45f2-b403-707161fe4002 is DONE. 0 Bytes processed. <a target=\"_blank\" href=\"https://console.cloud.google.com/bigquery?project=bigframes-dev&j=bq:us-east5:0b1b71d8-8546-45f2-b403-707161fe4002&page=queryresults\">Open Job</a>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
Expand Down Expand Up @@ -267,13 +229,15 @@
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"Query job 488a116f-44b2-4ff7-9f95-bd36473dab0f is DONE. 0 Bytes processed. <a target=\"_blank\" href=\"https://console.cloud.google.com/bigquery?project=bigframes-dev&j=bq:us-east5:488a116f-44b2-4ff7-9f95-bd36473dab0f&page=queryresults\">Open Job</a>"
"\n",
" Query processed 0 Bytes in a moment of slot time. [<a target=\"_blank\" href=\"https://console.cloud.google.com/bigquery?project=bigframes-dev&j=bq:us-east5:9bc70627-6891-44a4-b7d7-8a28e213cdec&page=queryresults\">Job bigframes-dev:us-east5.9bc70627-6891-44a4-b7d7-8a28e213cdec details</a>]\n",
" "
],
"text/plain": [
"<IPython.core.display.HTML object>"
Expand Down Expand Up @@ -301,24 +265,24 @@
" \"content\": message,\n",
" }\n",
" ],\n",
" model=\"claude-3-5-sonnet@20240620\",\n",
" model=\"claude-3-haiku@20240307\",\n",
" )\n",
" content_text = message.content[0].text if message.content else \"\"\n",
" return content_text"
]
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'bigframes-dev._b52b272a35b88e236e1f96fbe3f560c83a8fee85.bigframes_session265649_de1176dd4c57f40ba959503af3981682'"
"'bigframes-dev._e9a5162ae4daa9f50fda3f95febaa9781131f3b8.bigframes_sessionc10c73_49262141176cbf70037559ae84e834d3'"
]
},
"execution_count": 7,
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -330,16 +294,16 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 7,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'projects/bigframes-dev/locations/us-east5/functions/bigframes-session265649-de1176dd4c57f40ba959503af3981682'"
"'projects/bigframes-dev/locations/us-east5/functions/bigframes-sessionc10c73-49262141176cbf70037559ae84e834d3'"
]
},
"execution_count": 8,
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -351,49 +315,40 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 8,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"Query job 58b230a8-6536-4bac-ab02-dcf574692dd6 is DONE. 0 Bytes processed. <a target=\"_blank\" href=\"https://console.cloud.google.com/bigquery?project=bigframes-dev&j=bq:us-east5:58b230a8-6536-4bac-ab02-dcf574692dd6&page=queryresults\">Open Job</a>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"Query job 46d6a1e9-426a-4615-8eb5-98d34d08ec07 is DONE. 1.3 kB processed. <a target=\"_blank\" href=\"https://console.cloud.google.com/bigquery?project=bigframes-dev&j=bq:us-east5:46d6a1e9-426a-4615-8eb5-98d34d08ec07&page=queryresults\">Open Job</a>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"Query job cf8fcbaa-b233-47cd-b4e3-60876b24879f is DONE. 0 Bytes processed. <a target=\"_blank\" href=\"https://console.cloud.google.com/bigquery?project=bigframes-dev&j=bq:us-east5:cf8fcbaa-b233-47cd-b4e3-60876b24879f&page=queryresults\">Open Job</a>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"Query job 2620a544-d1df-4b30-bec4-4221e79ddf0f is DONE. 1.4 kB processed. <a target=\"_blank\" href=\"https://console.cloud.google.com/bigquery?project=bigframes-dev&j=bq:us-east5:2620a544-d1df-4b30-bec4-4221e79ddf0f&page=queryresults\">Open Job</a>"
"\n",
" Query started with request ID bigframes-dev:us-east5.821579f4-63ea-4072-a3ce-318e43768432.<details><summary>SQL</summary><pre>SELECT\n",
"`bfuid_col_3` AS `bfuid_col_3`,\n",
"`bfuid_col_4` AS `bfuid_col_4`,\n",
"`bfuid_col_5` AS `bfuid_col_5`\n",
"FROM\n",
"(SELECT\n",
" `t1`.`bfuid_col_3`,\n",
" `t1`.`bfuid_col_4`,\n",
" `t1`.`bfuid_col_5`,\n",
" `t1`.`bfuid_col_6` AS `bfuid_col_7`\n",
"FROM (\n",
" SELECT\n",
" `t0`.`level_0`,\n",
" `t0`.`column_0`,\n",
" `t0`.`bfuid_col_6`,\n",
" `t0`.`level_0` AS `bfuid_col_3`,\n",
" `t0`.`column_0` AS `bfuid_col_4`,\n",
" `bigframes-dev._e9a5162ae4daa9f50fda3f95febaa9781131f3b8.bigframes_sessionc10c73_49262141176cbf70037559ae84e834d3`(`t0`.`column_0`) AS `bfuid_col_5`\n",
" FROM (\n",
" SELECT\n",
" *\n",
" FROM UNNEST(ARRAY&lt;STRUCT&lt;`level_0` INT64, `column_0` STRING, `bfuid_col_6` INT64&gt;&gt;[STRUCT(0, &#x27;What is the capital of France?&#x27;, 0), STRUCT(1, &#x27;Explain the concept of photosynthesis in simple terms.&#x27;, 1), STRUCT(2, &#x27;Write a haiku about artificial intelligence.&#x27;, 2)]) AS `level_0`\n",
" ) AS `t0`\n",
") AS `t1`)\n",
"ORDER BY `bfuid_col_7` ASC NULLS LAST\n",
"LIMIT 10</pre></details>\n",
" "
],
"text/plain": [
"<IPython.core.display.HTML object>"
Expand Down Expand Up @@ -436,13 +391,12 @@
" <tr>\n",
" <th>1</th>\n",
" <td>Explain the concept of photosynthesis in simpl...</td>\n",
" <td>Photosynthesis is the process plants use to ma...</td>\n",
" <td>Photosynthesis is the process by which plants ...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>Write a haiku about artificial intelligence.</td>\n",
" <td>Here's a haiku about artificial intelligence:\n",
"...</td>\n",
" <td>Here is a haiku about artificial intelligence:...</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
Expand All @@ -457,14 +411,13 @@
"\n",
" answers \n",
"0 The capital of France is Paris. \n",
"1 Photosynthesis is the process plants use to ma... \n",
"2 Here's a haiku about artificial intelligence:\n",
"... \n",
"1 Photosynthesis is the process by which plants ... \n",
"2 Here is a haiku about artificial intelligence:... \n",
"\n",
"[3 rows x 2 columns]"
]
},
"execution_count": 9,
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -484,17 +437,30 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 9,
"metadata": {},
"outputs": [],
"outputs": [
{
"data": {
"text/html": [
"Session sessionc10c73 closed."
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"bpd.close_session()"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "venv",
"display_name": "venv (3.14.2)",
"language": "python",
"name": "python3"
},
Expand All @@ -508,7 +474,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.9"
"version": "3.14.2"
}
},
"nbformat": 4,
Expand Down
44 changes: 9 additions & 35 deletions tests/system/load/test_llm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,18 +98,10 @@ def test_llm_gemini_w_ground_with_google_search(llm_remote_text_df):


# (b/366290533): Claude models are of extremely low capacity. The tests should reside in small tests. Moving these here just to protect BQML's shared capacity(as load test only runs once per day.) and make sure we still have minimum coverage.
@pytest.mark.parametrize(
"model_name",
("claude-3-haiku", "claude-3-5-sonnet"),
)
@pytest.mark.flaky(retries=3, delay=120)
def test_claude3_text_generator_create_load(
dataset_id, model_name, session, session_us_east5, bq_connection
):
if model_name in ("claude-3-5-sonnet",):
session = session_us_east5
def test_claude3_text_generator_create_load(dataset_id, session, bq_connection):
claude3_text_generator_model = llm.Claude3TextGenerator(
model_name=model_name, connection_name=bq_connection, session=session
model_name="claude-3-haiku", connection_name=bq_connection, session=session
)
assert claude3_text_generator_model is not None
assert claude3_text_generator_model._bqml_model is not None
Expand All @@ -120,40 +112,28 @@ def test_claude3_text_generator_create_load(
)
assert f"{dataset_id}.temp_text_model" == reloaded_model._bqml_model.model_name
assert reloaded_model.connection_name == bq_connection
assert reloaded_model.model_name == model_name
assert reloaded_model.model_name == "claude-3-haiku"


@pytest.mark.parametrize(
"model_name",
("claude-3-haiku", "claude-3-5-sonnet"),
)
@pytest.mark.flaky(retries=3, delay=120)
def test_claude3_text_generator_predict_default_params_success(
llm_text_df, model_name, session, session_us_east5, bq_connection
llm_text_df, session, bq_connection
):
if model_name in ("claude-3-5-sonnet",):
session = session_us_east5
claude3_text_generator_model = llm.Claude3TextGenerator(
model_name=model_name, connection_name=bq_connection, session=session
model_name="claude-3-haiku", connection_name=bq_connection, session=session
)
df = claude3_text_generator_model.predict(llm_text_df).to_pandas()
utils.check_pandas_df_schema_and_index(
df, columns=utils.ML_GENERATE_TEXT_OUTPUT, index=3, col_exact=False
)


@pytest.mark.parametrize(
"model_name",
("claude-3-haiku", "claude-3-5-sonnet"),
)
@pytest.mark.flaky(retries=3, delay=120)
def test_claude3_text_generator_predict_with_params_success(
llm_text_df, model_name, session, session_us_east5, bq_connection
llm_text_df, session, bq_connection
):
if model_name in ("claude-3-5-sonnet",):
session = session_us_east5
claude3_text_generator_model = llm.Claude3TextGenerator(
model_name=model_name, connection_name=bq_connection, session=session
model_name="claude-3-haiku", connection_name=bq_connection, session=session
)
df = claude3_text_generator_model.predict(
llm_text_df, max_output_tokens=100, top_k=20, top_p=0.5
Expand All @@ -163,20 +143,14 @@ def test_claude3_text_generator_predict_with_params_success(
)


@pytest.mark.parametrize(
"model_name",
("claude-3-haiku", "claude-3-5-sonnet"),
)
@pytest.mark.flaky(retries=3, delay=120)
def test_claude3_text_generator_predict_multi_col_success(
llm_text_df, model_name, session, session_us_east5, bq_connection
llm_text_df, session, bq_connection
):
if model_name in ("claude-3-5-sonnet",):
session = session_us_east5

llm_text_df["additional_col"] = 1
claude3_text_generator_model = llm.Claude3TextGenerator(
model_name=model_name, connection_name=bq_connection, session=session
model_name="claude-3-haiku", connection_name=bq_connection, session=session
)
df = claude3_text_generator_model.predict(llm_text_df).to_pandas()
utils.check_pandas_df_schema_and_index(
Expand Down
Loading