From 0c3b23b206ae984db88387d4913d3c1a41cd58ca Mon Sep 17 00:00:00 2001 From: MarvinBeGood <46404767+MarvinBeGood@users.noreply.github.com> Date: Thu, 12 Mar 2026 15:16:46 +0100 Subject: [PATCH 1/3] Delete sagemaker-train/src/sagemaker/train/evaluate/Resolved template parameters: {'role_arn.md remove invalid file --- ...esolved template parameters: {'role_arn.md | 172 ------------------ 1 file changed, 172 deletions(-) delete mode 100644 sagemaker-train/src/sagemaker/train/evaluate/Resolved template parameters: {'role_arn.md diff --git a/sagemaker-train/src/sagemaker/train/evaluate/Resolved template parameters: {'role_arn.md b/sagemaker-train/src/sagemaker/train/evaluate/Resolved template parameters: {'role_arn.md deleted file mode 100644 index 9c1077203e..0000000000 --- a/sagemaker-train/src/sagemaker/train/evaluate/Resolved template parameters: {'role_arn.md +++ /dev/null @@ -1,172 +0,0 @@ -Resolved template parameters: {'role_arn': base_evaluator.py:757 - 'arn:aws:iam::634683118556:role/service-role/AmazonSageMaker-Exe - cutionRole-20251116T174807', 'mlflow_resource_arn': - 'arn:aws:sagemaker:us-east-1:634683118556:mlflow-app/app-DA25Q2S - 35KHZ', 'mlflow_experiment_name': None, 'mlflow_run_name': None, - 'model_package_group_arn': - 'arn:aws:sagemaker:us-east-1:634683118556:model-package-group/tm - p-humanlike-llama32-rlaif', 'source_model_package_arn': None, - 'base_model_arn': - 'arn:aws:sagemaker:us-east-1:aws:hub-content/SageMakerPublicHub/ - Model/meta-textgeneration-llama-3-2-1b-instruct/1.25.0', - 's3_output_path': - 's3://sagemaker-us-east-1-634683118556/tmp-humanlike-llama32-rla - if/eval', 'dataset_artifact_arn': - 'arn:aws:sagemaker:us-east-1:634683118556:artifact/c3c6611071894 - bad6a7f0925a729b02e', 'action_arn_prefix': - 'arn:aws:sagemaker:us-east-1:634683118556:action', - 'dataset_uri': - 'arn:aws:sagemaker:us-east-1:634683118556:hub-content/CKO4ACGI3U - RQBOO74C9JPLUMQNG02M2I4CIM9M931SQHE0625A30/DataSet/tmp-humanlike - -rlaif-eval/0.0.1', 'judge_model_id': - 'anthropic.claude-3-5-sonnet-20240620-v1:0', 'llmaj_metrics': - '[]', 'custom_metrics_s3_path': - 's3://sagemaker-us-east-1-634683118556/tmp-humanlike-llama32-rla - if/eval/evaluationinputs/eval-meta-1517aa3320251202-011237/custo - m-metrics.json', 'max_new_tokens': '8192', 'temperature': '0', - 'top_k': '-1', 'top_p': '1.0', 'pipeline_name': - 'SagemakerModelEvaluationType2-llmaj', 'evaluate_base_model': - True} - INFO Rendered pipeline definition: base_evaluator.py:766 - { - "Version": "2020-12-01", - "Metadata": {}, - "MlflowConfig": { - "MlflowResourceArn": - "arn:aws:sagemaker:us-east-1:634683118556:mlflow-app/app-DA25Q2S - 35KHZ" - }, - "Parameters": [], - "Steps": [ - { - "Name": "EvaluateBaseInferenceModel", - "Type": "Training", - "Arguments": { - "TrainingJobName": "BaseInference", - "RoleArn": - "arn:aws:iam::634683118556:role/service-role/AmazonSageMaker-Exe - cutionRole-20251116T174807", - "ServerlessJobConfig": { - "BaseModelArn": - "arn:aws:sagemaker:us-east-1:aws:hub-content/SageMakerPublicHub/ - Model/meta-textgeneration-llama-3-2-1b-instruct/1.25.0", - "AcceptEula": true, - "JobType": "Evaluation", - "EvaluationType": "BenchmarkEvaluation" - }, - "StoppingCondition": { - "MaxRuntimeInSeconds": 86400 - }, - "HyperParameters": { - "name": "BaseInference", - "task": "inference_only" - }, - "OutputDataConfig": { - "S3OutputPath": - "s3://sagemaker-us-east-1-634683118556/tmp-humanlike-llama32-rla - if/eval", - "CompressionType": "NONE" - }, - "InputDataConfig": [ - { - "ChannelName": "train", - "DataSource": { - "DatasetSource": { - "DatasetArn": - "arn:aws:sagemaker:us-east-1:634683118556:hub-content/CKO4ACGI3U - RQBOO74C9JPLUMQNG02M2I4CIM9M931SQHE0625A30/DataSet/tmp-humanlike - -rlaif-eval/0.0.1" - } - } - } - ] - } - }, - { - "Name": "EvaluateBaseModelMetrics", - "Type": "Training", - "DependsOn": [ - "EvaluateBaseInferenceModel" - ], - "Arguments": { - "TrainingJobName": { - "Std:Join": { - "On": "-", - "Values": [ - "base-llmaj-eval", - { - "Get": "Execution.PipelineExecutionId" - } - ] - } - }, - "RoleArn": - "arn:aws:iam::634683118556:role/service-role/AmazonSageMaker-Exe - cutionRole-20251116T174807", - "ServerlessJobConfig": { - "BaseModelArn": - "arn:aws:sagemaker:us-east-1:aws:hub-content/SageMakerPublicHub/ - Model/meta-textgeneration-llama-3-2-1b-instruct/1.25.0", - "AcceptEula": true, - "JobType": "Evaluation", - "EvaluationType": "LLMAJEvaluation" - }, - "StoppingCondition": { - "MaxRuntimeInSeconds": 86400 - }, - "HyperParameters": { - "name": { - "Std:Join": { - "On": "-", - "Values": [ - "base-llmaj-eval", - { - "Get": "Execution.PipelineExecutionId" - } - ] - } - }, - "judge_model_id": - "anthropic.claude-3-5-sonnet-20240620-v1:0", - "inference_data_s3_path": { - "Std:Join": { - "On": "", - "Values": [ - { - "Get": - "Steps.EvaluateBaseInferenceModel.OutputDataConfig.S3OutputPath" - }, - "/", - { - "Get": - "Steps.EvaluateBaseInferenceModel.TrainingJobName" - }, - "/output/output/", - "BaseInference", - "/eval_results/inference_output.jsonl" - ] - } - }, - "output_path": - "s3://sagemaker-us-east-1-634683118556/tmp-humanlike-llama32-rla - if/eval", - "llmaj_metrics": "[]", - "custom_metrics_s3_path": - "s3://sagemaker-us-east-1-634683118556/tmp-humanlike-llama32-rla - if/eval/evaluationinputs/eval-meta-1517aa3320251202-011237/custo - m-metrics.json", - "max_new_tokens": "8192", - "temperature": "0", - "top_k": "-1", - "top_p": "1.0" - }, - "OutputDataConfig": { - "S3OutputPath": - "s3://sagemaker-us-east-1-634683118556/tmp-humanlike-llama32-rla - if/eval", - "CompressionType": "NONE" - } - } - } - ] - } \ No newline at end of file From e4164837679a27b9fd92e5f3f08aff8df0837b5f Mon Sep 17 00:00:00 2001 From: MarvinBeGood Date: Sun, 15 Mar 2026 10:48:49 +0100 Subject: [PATCH 2/3] Refactor output data configuration handling in tuner --- sagemaker-train/src/sagemaker/train/tuner.py | 12 ++---------- sagemaker-train/tests/unit/train/test_tuner.py | 1 + 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/sagemaker-train/src/sagemaker/train/tuner.py b/sagemaker-train/src/sagemaker/train/tuner.py index f99f055f0b..aa0d68c5e8 100644 --- a/sagemaker-train/src/sagemaker/train/tuner.py +++ b/sagemaker-train/src/sagemaker/train/tuner.py @@ -253,6 +253,7 @@ def __init__( self.instance_configs_dict = None self.instance_configs = None self.autotune = autotune + self.output_data_config = model_trainer.output_data_config def override_resource_config( self, @@ -1331,7 +1332,6 @@ def _build_training_job_definition(self, inputs): from sagemaker.core.shapes import ( HyperParameterTrainingJobDefinition, HyperParameterAlgorithmSpecification, - OutputDataConfig, ResourceConfig, StoppingCondition, Channel, @@ -1422,14 +1422,6 @@ def _build_training_job_definition(self, inputs): if not any(c.channel_name == channel.channel_name for c in input_data_config): input_data_config.append(channel) - # Build output data config - output_config = OutputDataConfig( - s3_output_path=( - model_trainer.output_data_config.s3_output_path - if model_trainer.output_data_config - else None - ) - ) # Build resource config resource_config = ResourceConfig( @@ -1456,7 +1448,7 @@ def _build_training_job_definition(self, inputs): algorithm_specification=algorithm_spec, role_arn=model_trainer.role, input_data_config=input_data_config if input_data_config else None, - output_data_config=output_config, + output_data_config=self.output_data_config, resource_config=resource_config, stopping_condition=stopping_condition, static_hyper_parameters=self.static_hyperparameters or {}, diff --git a/sagemaker-train/tests/unit/train/test_tuner.py b/sagemaker-train/tests/unit/train/test_tuner.py index c0255eac47..04a676644d 100644 --- a/sagemaker-train/tests/unit/train/test_tuner.py +++ b/sagemaker-train/tests/unit/train/test_tuner.py @@ -53,6 +53,7 @@ def _create_mock_model_trainer(with_internal_channels=False): trainer.training_input_mode = "File" trainer.role = "arn:aws:iam::123456789012:role/SageMakerRole" trainer.output_data_config = MagicMock() + trainer.output_data_config.kms_key_id = None trainer.output_data_config.s3_output_path = "s3://bucket/output" trainer.compute = MagicMock() trainer.compute.instance_type = "ml.m5.xlarge" From fad349c12c169d6346dfa7687a9b5a47942a5423 Mon Sep 17 00:00:00 2001 From: MarvinBeGood Date: Sun, 15 Mar 2026 11:30:06 +0100 Subject: [PATCH 3/3] renamed Resolved template parameters: {'role_arn.md to Resolved_template_parameters_role_arn.md to ensure the filename is also valid for windows --- .../Resolved_template_parameters_role_arn.md | 172 ++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 sagemaker-train/src/sagemaker/train/evaluate/Resolved_template_parameters_role_arn.md diff --git a/sagemaker-train/src/sagemaker/train/evaluate/Resolved_template_parameters_role_arn.md b/sagemaker-train/src/sagemaker/train/evaluate/Resolved_template_parameters_role_arn.md new file mode 100644 index 0000000000..33342ee90c --- /dev/null +++ b/sagemaker-train/src/sagemaker/train/evaluate/Resolved_template_parameters_role_arn.md @@ -0,0 +1,172 @@ +Resolved template parameters: {'role_arn': base_evaluator.py:757 + 'arn:aws:iam::634683118556:role/service-role/AmazonSageMaker-Exe + cutionRole-20251116T174807', 'mlflow_resource_arn': + 'arn:aws:sagemaker:us-east-1:634683118556:mlflow-app/app-DA25Q2S + 35KHZ', 'mlflow_experiment_name': None, 'mlflow_run_name': None, + 'model_package_group_arn': + 'arn:aws:sagemaker:us-east-1:634683118556:model-package-group/tm + p-humanlike-llama32-rlaif', 'source_model_package_arn': None, + 'base_model_arn': + 'arn:aws:sagemaker:us-east-1:aws:hub-content/SageMakerPublicHub/ + Model/meta-textgeneration-llama-3-2-1b-instruct/1.25.0', + 's3_output_path': + 's3://sagemaker-us-east-1-634683118556/tmp-humanlike-llama32-rla + if/eval', 'dataset_artifact_arn': + 'arn:aws:sagemaker:us-east-1:634683118556:artifact/c3c6611071894 + bad6a7f0925a729b02e', 'action_arn_prefix': + 'arn:aws:sagemaker:us-east-1:634683118556:action', + 'dataset_uri': + 'arn:aws:sagemaker:us-east-1:634683118556:hub-content/CKO4ACGI3U + RQBOO74C9JPLUMQNG02M2I4CIM9M931SQHE0625A30/DataSet/tmp-humanlike + -rlaif-eval/0.0.1', 'judge_model_id': + 'anthropic.claude-3-5-sonnet-20240620-v1:0', 'llmaj_metrics': + '[]', 'custom_metrics_s3_path': + 's3://sagemaker-us-east-1-634683118556/tmp-humanlike-llama32-rla + if/eval/evaluationinputs/eval-meta-1517aa3320251202-011237/custo + m-metrics.json', 'max_new_tokens': '8192', 'temperature': '0', + 'top_k': '-1', 'top_p': '1.0', 'pipeline_name': + 'SagemakerModelEvaluationType2-llmaj', 'evaluate_base_model': + True} + INFO Rendered pipeline definition: base_evaluator.py:766 + { + "Version": "2020-12-01", + "Metadata": {}, + "MlflowConfig": { + "MlflowResourceArn": + "arn:aws:sagemaker:us-east-1:634683118556:mlflow-app/app-DA25Q2S + 35KHZ" + }, + "Parameters": [], + "Steps": [ + { + "Name": "EvaluateBaseInferenceModel", + "Type": "Training", + "Arguments": { + "TrainingJobName": "BaseInference", + "RoleArn": + "arn:aws:iam::634683118556:role/service-role/AmazonSageMaker-Exe + cutionRole-20251116T174807", + "ServerlessJobConfig": { + "BaseModelArn": + "arn:aws:sagemaker:us-east-1:aws:hub-content/SageMakerPublicHub/ + Model/meta-textgeneration-llama-3-2-1b-instruct/1.25.0", + "AcceptEula": true, + "JobType": "Evaluation", + "EvaluationType": "BenchmarkEvaluation" + }, + "StoppingCondition": { + "MaxRuntimeInSeconds": 86400 + }, + "HyperParameters": { + "name": "BaseInference", + "task": "inference_only" + }, + "OutputDataConfig": { + "S3OutputPath": + "s3://sagemaker-us-east-1-634683118556/tmp-humanlike-llama32-rla + if/eval", + "CompressionType": "NONE" + }, + "InputDataConfig": [ + { + "ChannelName": "train", + "DataSource": { + "DatasetSource": { + "DatasetArn": + "arn:aws:sagemaker:us-east-1:634683118556:hub-content/CKO4ACGI3U + RQBOO74C9JPLUMQNG02M2I4CIM9M931SQHE0625A30/DataSet/tmp-humanlike + -rlaif-eval/0.0.1" + } + } + } + ] + } + }, + { + "Name": "EvaluateBaseModelMetrics", + "Type": "Training", + "DependsOn": [ + "EvaluateBaseInferenceModel" + ], + "Arguments": { + "TrainingJobName": { + "Std:Join": { + "On": "-", + "Values": [ + "base-llmaj-eval", + { + "Get": "Execution.PipelineExecutionId" + } + ] + } + }, + "RoleArn": + "arn:aws:iam::634683118556:role/service-role/AmazonSageMaker-Exe + cutionRole-20251116T174807", + "ServerlessJobConfig": { + "BaseModelArn": + "arn:aws:sagemaker:us-east-1:aws:hub-content/SageMakerPublicHub/ + Model/meta-textgeneration-llama-3-2-1b-instruct/1.25.0", + "AcceptEula": true, + "JobType": "Evaluation", + "EvaluationType": "LLMAJEvaluation" + }, + "StoppingCondition": { + "MaxRuntimeInSeconds": 86400 + }, + "HyperParameters": { + "name": { + "Std:Join": { + "On": "-", + "Values": [ + "base-llmaj-eval", + { + "Get": "Execution.PipelineExecutionId" + } + ] + } + }, + "judge_model_id": + "anthropic.claude-3-5-sonnet-20240620-v1:0", + "inference_data_s3_path": { + "Std:Join": { + "On": "", + "Values": [ + { + "Get": + "Steps.EvaluateBaseInferenceModel.OutputDataConfig.S3OutputPath" + }, + "/", + { + "Get": + "Steps.EvaluateBaseInferenceModel.TrainingJobName" + }, + "/output/output/", + "BaseInference", + "/eval_results/inference_output.jsonl" + ] + } + }, + "output_path": + "s3://sagemaker-us-east-1-634683118556/tmp-humanlike-llama32-rla + if/eval", + "llmaj_metrics": "[]", + "custom_metrics_s3_path": + "s3://sagemaker-us-east-1-634683118556/tmp-humanlike-llama32-rla + if/eval/evaluationinputs/eval-meta-1517aa3320251202-011237/custo + m-metrics.json", + "max_new_tokens": "8192", + "temperature": "0", + "top_k": "-1", + "top_p": "1.0" + }, + "OutputDataConfig": { + "S3OutputPath": + "s3://sagemaker-us-east-1-634683118556/tmp-humanlike-llama32-rla + if/eval", + "CompressionType": "NONE" + } + } + } + ] + }