diff --git a/bigframes/ml/llm.py b/bigframes/ml/llm.py index 585599c9b6c..68842961e3f 100644 --- a/bigframes/ml/llm.py +++ b/bigframes/ml/llm.py @@ -877,7 +877,7 @@ class Claude3TextGenerator(base.RetriableRemotePredictor): The model for natural language tasks. Possible values are "claude-3-sonnet", "claude-3-haiku", "claude-3-5-sonnet" and "claude-3-opus". "claude-3-sonnet" (deprecated) is Anthropic's dependable combination of skills and speed. It is engineered to be dependable for scaled AI deployments across a variety of use cases. "claude-3-haiku" is Anthropic's fastest, most compact vision and text model for near-instant responses to simple queries, meant for seamless AI experiences mimicking human interactions. - "claude-3-5-sonnet" is Anthropic's most powerful AI model and maintains the speed and cost of Claude 3 Sonnet, which is a mid-tier model. + "claude-3-5-sonnet" (deprecated) is Anthropic's most powerful AI model and maintains the speed and cost of Claude 3 Sonnet, which is a mid-tier model. "claude-3-opus" (deprecated) is Anthropic's second-most powerful AI model, with strong performance on highly complex tasks. https://cloud.google.com/vertex-ai/generative-ai/docs/partner-models/use-claude#available-claude-models If no setting is provided, "claude-3-sonnet" will be used by default diff --git a/notebooks/remote_functions/remote_function_vertex_claude_model.ipynb b/notebooks/remote_functions/remote_function_vertex_claude_model.ipynb index 9792c90205c..017ed596a2f 100644 --- a/notebooks/remote_functions/remote_function_vertex_claude_model.ipynb +++ b/notebooks/remote_functions/remote_function_vertex_claude_model.ipynb @@ -28,20 +28,6 @@ "" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import sys\n", - "\n", - "# Python 3.13 is not yet a supported runtime for remote functions.\n", - "# See: https://cloud.google.com/functions/docs/runtime-support#python for the supported runtimes.\n", - "if sys.version_info >= (3, 13, 0):\n", - " sys.exit(0)" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -155,30 +141,6 @@ "execution_count": 4, "metadata": {}, "outputs": [ - { - "data": { - "text/html": [ - "Query job c4c27713-51c8-4293-8454-5c904df79318 is DONE. 0 Bytes processed. Open Job" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "Query job 0b1b71d8-8546-45f2-b403-707161fe4002 is DONE. 0 Bytes processed. Open Job" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, { "data": { "text/html": [ @@ -267,13 +229,15 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/html": [ - "Query job 488a116f-44b2-4ff7-9f95-bd36473dab0f is DONE. 0 Bytes processed. Open Job" + "\n", + " Query processed 0 Bytes in a moment of slot time. [Job bigframes-dev:us-east5.9bc70627-6891-44a4-b7d7-8a28e213cdec details]\n", + " " ], "text/plain": [ "" @@ -301,7 +265,7 @@ " \"content\": message,\n", " }\n", " ],\n", - " model=\"claude-3-5-sonnet@20240620\",\n", + " model=\"claude-3-haiku@20240307\",\n", " )\n", " content_text = message.content[0].text if message.content else \"\"\n", " return content_text" @@ -309,16 +273,16 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "'bigframes-dev._b52b272a35b88e236e1f96fbe3f560c83a8fee85.bigframes_session265649_de1176dd4c57f40ba959503af3981682'" + "'bigframes-dev._e9a5162ae4daa9f50fda3f95febaa9781131f3b8.bigframes_sessionc10c73_49262141176cbf70037559ae84e834d3'" ] }, - "execution_count": 7, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -330,16 +294,16 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "'projects/bigframes-dev/locations/us-east5/functions/bigframes-session265649-de1176dd4c57f40ba959503af3981682'" + "'projects/bigframes-dev/locations/us-east5/functions/bigframes-sessionc10c73-49262141176cbf70037559ae84e834d3'" ] }, - "execution_count": 8, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -351,49 +315,40 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 8, "metadata": {}, "outputs": [ { "data": { "text/html": [ - "Query job 58b230a8-6536-4bac-ab02-dcf574692dd6 is DONE. 0 Bytes processed. Open Job" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "Query job 46d6a1e9-426a-4615-8eb5-98d34d08ec07 is DONE. 1.3 kB processed. Open Job" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "Query job cf8fcbaa-b233-47cd-b4e3-60876b24879f is DONE. 0 Bytes processed. Open Job" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "Query job 2620a544-d1df-4b30-bec4-4221e79ddf0f is DONE. 1.4 kB processed. Open Job" + "\n", + " Query started with request ID bigframes-dev:us-east5.821579f4-63ea-4072-a3ce-318e43768432.
SQL
SELECT\n",
+       "`bfuid_col_3` AS `bfuid_col_3`,\n",
+       "`bfuid_col_4` AS `bfuid_col_4`,\n",
+       "`bfuid_col_5` AS `bfuid_col_5`\n",
+       "FROM\n",
+       "(SELECT\n",
+       "  `t1`.`bfuid_col_3`,\n",
+       "  `t1`.`bfuid_col_4`,\n",
+       "  `t1`.`bfuid_col_5`,\n",
+       "  `t1`.`bfuid_col_6` AS `bfuid_col_7`\n",
+       "FROM (\n",
+       "  SELECT\n",
+       "    `t0`.`level_0`,\n",
+       "    `t0`.`column_0`,\n",
+       "    `t0`.`bfuid_col_6`,\n",
+       "    `t0`.`level_0` AS `bfuid_col_3`,\n",
+       "    `t0`.`column_0` AS `bfuid_col_4`,\n",
+       "    `bigframes-dev._e9a5162ae4daa9f50fda3f95febaa9781131f3b8.bigframes_sessionc10c73_49262141176cbf70037559ae84e834d3`(`t0`.`column_0`) AS `bfuid_col_5`\n",
+       "  FROM (\n",
+       "    SELECT\n",
+       "      *\n",
+       "    FROM UNNEST(ARRAY<STRUCT<`level_0` INT64, `column_0` STRING, `bfuid_col_6` INT64>>[STRUCT(0, 'What is the capital of France?', 0), STRUCT(1, 'Explain the concept of photosynthesis in simple terms.', 1), STRUCT(2, 'Write a haiku about artificial intelligence.', 2)]) AS `level_0`\n",
+       "  ) AS `t0`\n",
+       ") AS `t1`)\n",
+       "ORDER BY `bfuid_col_7` ASC NULLS LAST\n",
+       "LIMIT 10
\n", + " " ], "text/plain": [ "" @@ -436,13 +391,12 @@ " \n", " 1\n", " Explain the concept of photosynthesis in simpl...\n", - " Photosynthesis is the process plants use to ma...\n", + " Photosynthesis is the process by which plants ...\n", " \n", " \n", " 2\n", " Write a haiku about artificial intelligence.\n", - " Here's a haiku about artificial intelligence:\n", - "...\n", + " Here is a haiku about artificial intelligence:...\n", " \n", " \n", "\n", @@ -457,14 +411,13 @@ "\n", " answers \n", "0 The capital of France is Paris. \n", - "1 Photosynthesis is the process plants use to ma... \n", - "2 Here's a haiku about artificial intelligence:\n", - "... \n", + "1 Photosynthesis is the process by which plants ... \n", + "2 Here is a haiku about artificial intelligence:... \n", "\n", "[3 rows x 2 columns]" ] }, - "execution_count": 9, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -484,9 +437,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "Session sessionc10c73 closed." + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "bpd.close_session()" ] @@ -494,7 +460,7 @@ ], "metadata": { "kernelspec": { - "display_name": "venv", + "display_name": "venv (3.14.2)", "language": "python", "name": "python3" }, @@ -508,7 +474,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.9" + "version": "3.14.2" } }, "nbformat": 4, diff --git a/tests/system/load/test_llm.py b/tests/system/load/test_llm.py index 25cde92c133..22a6f0bfd2e 100644 --- a/tests/system/load/test_llm.py +++ b/tests/system/load/test_llm.py @@ -98,18 +98,10 @@ def test_llm_gemini_w_ground_with_google_search(llm_remote_text_df): # (b/366290533): Claude models are of extremely low capacity. The tests should reside in small tests. Moving these here just to protect BQML's shared capacity(as load test only runs once per day.) and make sure we still have minimum coverage. -@pytest.mark.parametrize( - "model_name", - ("claude-3-haiku", "claude-3-5-sonnet"), -) @pytest.mark.flaky(retries=3, delay=120) -def test_claude3_text_generator_create_load( - dataset_id, model_name, session, session_us_east5, bq_connection -): - if model_name in ("claude-3-5-sonnet",): - session = session_us_east5 +def test_claude3_text_generator_create_load(dataset_id, session, bq_connection): claude3_text_generator_model = llm.Claude3TextGenerator( - model_name=model_name, connection_name=bq_connection, session=session + model_name="claude-3-haiku", connection_name=bq_connection, session=session ) assert claude3_text_generator_model is not None assert claude3_text_generator_model._bqml_model is not None @@ -120,21 +112,15 @@ def test_claude3_text_generator_create_load( ) assert f"{dataset_id}.temp_text_model" == reloaded_model._bqml_model.model_name assert reloaded_model.connection_name == bq_connection - assert reloaded_model.model_name == model_name + assert reloaded_model.model_name == "claude-3-haiku" -@pytest.mark.parametrize( - "model_name", - ("claude-3-haiku", "claude-3-5-sonnet"), -) @pytest.mark.flaky(retries=3, delay=120) def test_claude3_text_generator_predict_default_params_success( - llm_text_df, model_name, session, session_us_east5, bq_connection + llm_text_df, session, bq_connection ): - if model_name in ("claude-3-5-sonnet",): - session = session_us_east5 claude3_text_generator_model = llm.Claude3TextGenerator( - model_name=model_name, connection_name=bq_connection, session=session + model_name="claude-3-haiku", connection_name=bq_connection, session=session ) df = claude3_text_generator_model.predict(llm_text_df).to_pandas() utils.check_pandas_df_schema_and_index( @@ -142,18 +128,12 @@ def test_claude3_text_generator_predict_default_params_success( ) -@pytest.mark.parametrize( - "model_name", - ("claude-3-haiku", "claude-3-5-sonnet"), -) @pytest.mark.flaky(retries=3, delay=120) def test_claude3_text_generator_predict_with_params_success( - llm_text_df, model_name, session, session_us_east5, bq_connection + llm_text_df, session, bq_connection ): - if model_name in ("claude-3-5-sonnet",): - session = session_us_east5 claude3_text_generator_model = llm.Claude3TextGenerator( - model_name=model_name, connection_name=bq_connection, session=session + model_name="claude-3-haiku", connection_name=bq_connection, session=session ) df = claude3_text_generator_model.predict( llm_text_df, max_output_tokens=100, top_k=20, top_p=0.5 @@ -163,20 +143,14 @@ def test_claude3_text_generator_predict_with_params_success( ) -@pytest.mark.parametrize( - "model_name", - ("claude-3-haiku", "claude-3-5-sonnet"), -) @pytest.mark.flaky(retries=3, delay=120) def test_claude3_text_generator_predict_multi_col_success( - llm_text_df, model_name, session, session_us_east5, bq_connection + llm_text_df, session, bq_connection ): - if model_name in ("claude-3-5-sonnet",): - session = session_us_east5 llm_text_df["additional_col"] = 1 claude3_text_generator_model = llm.Claude3TextGenerator( - model_name=model_name, connection_name=bq_connection, session=session + model_name="claude-3-haiku", connection_name=bq_connection, session=session ) df = claude3_text_generator_model.predict(llm_text_df).to_pandas() utils.check_pandas_df_schema_and_index(