From 81d06b99e098ead24d4c935bd538c36de0909226 Mon Sep 17 00:00:00 2001 From: Eli Davidson Date: Mon, 3 Aug 2026 19:15:37 +0000 Subject: [PATCH 1/5] fix(serve): propagate additional_model_data_sources for JumpStart models ModelBuilder._build_for_jumpstart() copied image_uri, env, enable_network_isolation, model_reference_arn, and model_data from the resolved JumpStart init_kwargs, but never propagated additional_model_data_sources. Any JumpStart model whose spec declares an extra S3 data channel (e.g. an EAGLE speculative decoding draft model, a LoRA adapter, or a tokenizer override) therefore had that channel silently dropped from the CreateModel call, and the container failed at runtime trying to read artifacts from a path that was never mounted. Propagate additional_model_data_sources from init_kwargs onto the builder so _prepare_container_def_base attaches it to the CreateModel request. Each source is filtered to only the keys the CreateModel API accepts (ChannelName, S3DataSource); JumpStart specs may include extra metadata keys such as HostingEulaKey that the API rejects. Adds unit tests for both the propagation+filtering path and the no-op case when the spec declares no additional data sources. --- .../sagemaker/serve/model_builder_servers.py | 18 +++++ .../servers/test_model_builder_servers.py | 69 +++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/sagemaker-serve/src/sagemaker/serve/model_builder_servers.py b/sagemaker-serve/src/sagemaker/serve/model_builder_servers.py index 08a77e0d1e..f39aa51bba 100644 --- a/sagemaker-serve/src/sagemaker/serve/model_builder_servers.py +++ b/sagemaker-serve/src/sagemaker/serve/model_builder_servers.py @@ -1014,6 +1014,24 @@ def _build_for_jumpstart(self) -> Model: if getattr(init_kwargs, "model_reference_arn", None): self.model_reference_arn = init_kwargs.model_reference_arn + # Propagate additional model data sources from the JumpStart spec (e.g. + # speculative decoding draft models like EAGLE, LoRA adapters, or any + # extra S3 data channel). Without this, sources declared in the spec are + # silently dropped from the CreateModel call and the container fails to + # find the referenced artifacts at runtime. + additional_model_data_sources = getattr( + init_kwargs, "additional_model_data_sources", None + ) + if isinstance(additional_model_data_sources, list) and additional_model_data_sources: + # Filter each source to only the keys the CreateModel API accepts. + # JumpStart specs may include extra metadata keys (e.g. HostingEulaKey) + # that the API rejects. + _api_keys = {"ChannelName", "S3DataSource"} + self.additional_model_data_sources = [ + {key: value for key, value in source.items() if key in _api_keys} + for source in additional_model_data_sources + ] + # Handle model artifacts for fine-tuned models if hasattr(init_kwargs, "model_data") and init_kwargs.model_data: if ( diff --git a/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py b/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py index 4fc1a558ba..c9d284e860 100644 --- a/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py +++ b/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py @@ -1004,6 +1004,75 @@ def test_build_sagemaker_endpoint_djl(self, mock_create, mock_prepare, mock_init mock_create.assert_called_once() + @patch("sagemaker.core.jumpstart.factory.utils.get_init_kwargs") + @patch.object(MockModelBuilderServers, "_prepare_for_mode") + @patch.object(MockModelBuilderServers, "_create_model") + def test_build_propagates_additional_model_data_sources( + self, mock_create, mock_prepare, mock_init + ): + """additional_model_data_sources from the JumpStart spec are propagated + to the builder and filtered to only API-accepted keys.""" + mock_init_kwargs = Mock() + mock_init_kwargs.image_uri = "djl-inference:0.21.0" + mock_init_kwargs.env = {} + mock_init_kwargs.model_data = "s3://bucket/model.tar.gz" + # JumpStart spec includes a metadata key (HostingEulaKey) that the + # CreateModel API does not accept; it must be filtered out. + mock_init_kwargs.additional_model_data_sources = [ + { + "ChannelName": "draft_model", + "S3DataSource": { + "S3Uri": "s3://bucket/eagle/", + "S3DataType": "S3Prefix", + "CompressionType": "None", + }, + "HostingEulaKey": None, + } + ] + mock_init.return_value = mock_init_kwargs + mock_create.return_value = Mock() + self.builder.mode = Mode.SAGEMAKER_ENDPOINT + self.builder.image_uri = None + + self.builder._build_for_jumpstart() + + self.assertEqual( + self.builder.additional_model_data_sources, + [ + { + "ChannelName": "draft_model", + "S3DataSource": { + "S3Uri": "s3://bucket/eagle/", + "S3DataType": "S3Prefix", + "CompressionType": "None", + }, + } + ], + ) + + @patch("sagemaker.core.jumpstart.factory.utils.get_init_kwargs") + @patch.object(MockModelBuilderServers, "_prepare_for_mode") + @patch.object(MockModelBuilderServers, "_create_model") + def test_build_without_additional_model_data_sources( + self, mock_create, mock_prepare, mock_init + ): + """Models whose spec has no additional_model_data_sources are unaffected.""" + mock_init_kwargs = Mock() + mock_init_kwargs.image_uri = "djl-inference:0.21.0" + mock_init_kwargs.env = {} + mock_init_kwargs.model_data = "s3://bucket/model.tar.gz" + mock_init_kwargs.additional_model_data_sources = None + mock_init.return_value = mock_init_kwargs + mock_create.return_value = Mock() + self.builder.mode = Mode.SAGEMAKER_ENDPOINT + self.builder.image_uri = None + + self.builder._build_for_jumpstart() + + self.assertIsNone( + getattr(self.builder, "additional_model_data_sources", None) + ) + class TestDeployWrappers(unittest.TestCase): """Test deploy wrapper methods.""" From bc112ff22c345fc0686847f7376d04b073329e5e Mon Sep 17 00:00:00 2001 From: Eli Davidson Date: Tue, 4 Aug 2026 13:19:48 +0000 Subject: [PATCH 2/5] fix(serve): enforce accept_eula on gated additional model data sources A source carrying a truthy HostingEulaKey is gated and requires license acceptance, exactly like the base model. container_def folds accept_eula into the base model's ModelAccessConfig but passes additional sources through untouched, so the acceptance is translated onto each gated source here: fold ModelAccessConfig={AcceptEula: True} into its S3DataSource when accept_eula is set, raise the same error the base model raises when it is not. Ungated sources just drop the JumpStart-internal HostingEulaKey the CreateModel API rejects. Test coverage mirrors the v2 helper's case inventory: multiple public sources, ungated ignores acceptance, mixed public+gated, gated accept, gated missing/false acceptance raises, plus a raw-spec-JSON end-to-end test driving the real JumpStartModelSpecs parsing and factory shaping. --- .../sagemaker/serve/model_builder_servers.py | 47 ++- .../servers/test_model_builder_servers.py | 392 +++++++++++++++++- 2 files changed, 423 insertions(+), 16 deletions(-) diff --git a/sagemaker-serve/src/sagemaker/serve/model_builder_servers.py b/sagemaker-serve/src/sagemaker/serve/model_builder_servers.py index f39aa51bba..57f43d7504 100644 --- a/sagemaker-serve/src/sagemaker/serve/model_builder_servers.py +++ b/sagemaker-serve/src/sagemaker/serve/model_builder_servers.py @@ -1014,23 +1014,44 @@ def _build_for_jumpstart(self) -> Model: if getattr(init_kwargs, "model_reference_arn", None): self.model_reference_arn = init_kwargs.model_reference_arn - # Propagate additional model data sources from the JumpStart spec (e.g. - # speculative decoding draft models like EAGLE, LoRA adapters, or any - # extra S3 data channel). Without this, sources declared in the spec are - # silently dropped from the CreateModel call and the container fails to - # find the referenced artifacts at runtime. + # Propagate additional model data sources resolved from the JumpStart + # spec (e.g. speculative decoding draft models like EAGLE). The JumpStart + # factory (_add_additional_model_data_sources_to_kwargs) already returns + # these in CreateModel API shape via camel_case_to_pascal_case(...). + # + # A source carrying a truthy ``HostingEulaKey`` is gated and requires the + # user to accept its end-user license agreement, exactly like the base + # model. ``container_def`` folds ``accept_eula`` into the base model's + # ModelAccessConfig but passes additional sources through untouched, so + # the acceptance must be translated onto each gated source here: enforce + # ``accept_eula`` and move it into the source's ``ModelAccessConfig`` + # (mirroring _add_model_access_configs_to_model_data_sources and the base + # model's handling in _create_model). Ungated sources simply drop the + # JumpStart-internal ``HostingEulaKey`` metadata, which the CreateModel + # API rejects. Without any of this, sources declared in the spec are + # dropped from the CreateModel call and the container fails to find the + # referenced artifacts at runtime. additional_model_data_sources = getattr( init_kwargs, "additional_model_data_sources", None ) if isinstance(additional_model_data_sources, list) and additional_model_data_sources: - # Filter each source to only the keys the CreateModel API accepts. - # JumpStart specs may include extra metadata keys (e.g. HostingEulaKey) - # that the API rejects. - _api_keys = {"ChannelName", "S3DataSource"} - self.additional_model_data_sources = [ - {key: value for key, value in source.items() if key in _api_keys} - for source in additional_model_data_sources - ] + accept_eula = getattr(self, "accept_eula", None) + sanitized_sources = [] + for source in additional_model_data_sources: + sanitized_source = dict(source) + if sanitized_source.pop("HostingEulaKey", None): + if not accept_eula: + raise ValueError( + "accept_eula must be set to True to deploy this model. " + "Please set accept_eula=True on the ModelBuilder instance " + "to confirm you have read and accepted the end-user license " + "agreement for this model." + ) + s3_data_source = dict(sanitized_source.get("S3DataSource", {})) + s3_data_source["ModelAccessConfig"] = {"AcceptEula": True} + sanitized_source["S3DataSource"] = s3_data_source + sanitized_sources.append(sanitized_source) + self.additional_model_data_sources = sanitized_sources # Handle model artifacts for fine-tuned models if hasattr(init_kwargs, "model_data") and init_kwargs.model_data: diff --git a/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py b/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py index c9d284e860..03f8c908c4 100644 --- a/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py +++ b/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py @@ -1011,13 +1011,14 @@ def test_build_propagates_additional_model_data_sources( self, mock_create, mock_prepare, mock_init ): """additional_model_data_sources from the JumpStart spec are propagated - to the builder and filtered to only API-accepted keys.""" + to the builder with the JumpStart-internal HostingEulaKey metadata + stripped (mirroring _add_model_access_configs_to_model_data_sources).""" mock_init_kwargs = Mock() mock_init_kwargs.image_uri = "djl-inference:0.21.0" mock_init_kwargs.env = {} mock_init_kwargs.model_data = "s3://bucket/model.tar.gz" - # JumpStart spec includes a metadata key (HostingEulaKey) that the - # CreateModel API does not accept; it must be filtered out. + # JumpStart spec includes an internal metadata key (HostingEulaKey) that + # the CreateModel API does not accept; it must be stripped. mock_init_kwargs.additional_model_data_sources = [ { "ChannelName": "draft_model", @@ -1073,6 +1074,391 @@ def test_build_without_additional_model_data_sources( getattr(self.builder, "additional_model_data_sources", None) ) + @patch("sagemaker.core.jumpstart.factory.utils.get_init_kwargs") + @patch.object(MockModelBuilderServers, "_prepare_for_mode") + @patch.object(MockModelBuilderServers, "_create_model") + def test_build_gated_additional_model_data_source_with_accept_eula( + self, mock_create, mock_prepare, mock_init + ): + """A gated additional source (truthy HostingEulaKey) has the acceptance + folded into its ModelAccessConfig when accept_eula is set, mirroring the + base model's EULA handling.""" + mock_init_kwargs = Mock() + mock_init_kwargs.image_uri = "djl-inference:0.21.0" + mock_init_kwargs.env = {} + mock_init_kwargs.model_data = "s3://bucket/model.tar.gz" + mock_init_kwargs.additional_model_data_sources = [ + { + "ChannelName": "draft_model", + "S3DataSource": { + "S3Uri": "s3://bucket/gated/", + "S3DataType": "S3Prefix", + "CompressionType": "None", + }, + "HostingEulaKey": "fmhMetadata/eula/llama3Eula.txt", + } + ] + mock_init.return_value = mock_init_kwargs + mock_create.return_value = Mock() + self.builder.mode = Mode.SAGEMAKER_ENDPOINT + self.builder.image_uri = None + self.builder.accept_eula = True + + self.builder._build_for_jumpstart() + + self.assertEqual( + self.builder.additional_model_data_sources, + [ + { + "ChannelName": "draft_model", + "S3DataSource": { + "S3Uri": "s3://bucket/gated/", + "S3DataType": "S3Prefix", + "CompressionType": "None", + "ModelAccessConfig": {"AcceptEula": True}, + }, + } + ], + ) + + @patch("sagemaker.core.jumpstart.factory.utils.get_init_kwargs") + @patch.object(MockModelBuilderServers, "_prepare_for_mode") + @patch.object(MockModelBuilderServers, "_create_model") + def test_build_gated_additional_model_data_source_without_accept_eula_raises( + self, mock_create, mock_prepare, mock_init + ): + """A gated additional source without accept_eula raises, matching the + base model's EULA enforcement.""" + mock_init_kwargs = Mock() + mock_init_kwargs.image_uri = "djl-inference:0.21.0" + mock_init_kwargs.env = {} + mock_init_kwargs.model_data = "s3://bucket/model.tar.gz" + mock_init_kwargs.additional_model_data_sources = [ + { + "ChannelName": "draft_model", + "S3DataSource": { + "S3Uri": "s3://bucket/gated/", + "S3DataType": "S3Prefix", + "CompressionType": "None", + }, + "HostingEulaKey": "fmhMetadata/eula/llama3Eula.txt", + } + ] + mock_init.return_value = mock_init_kwargs + mock_create.return_value = Mock() + self.builder.mode = Mode.SAGEMAKER_ENDPOINT + self.builder.image_uri = None + self.builder.accept_eula = None + + with self.assertRaises(ValueError) as ctx: + self.builder._build_for_jumpstart() + self.assertIn("accept_eula must be set to True", str(ctx.exception)) + + @patch("sagemaker.core.jumpstart.factory.utils.get_init_kwargs") + @patch.object(MockModelBuilderServers, "_prepare_for_mode") + @patch.object(MockModelBuilderServers, "_create_model") + def test_build_multiple_public_additional_model_data_sources_pass_through( + self, mock_create, mock_prepare, mock_init + ): + """Multiple ungated additional sources are all propagated with only the + HostingEulaKey metadata stripped.""" + mock_init_kwargs = Mock() + mock_init_kwargs.image_uri = "djl-inference:0.21.0" + mock_init_kwargs.env = {} + mock_init_kwargs.model_data = "s3://bucket/model.tar.gz" + mock_init_kwargs.additional_model_data_sources = [ + { + "ChannelName": "draft_model", + "S3DataSource": { + "S3Uri": "s3://bucket/eagle/", + "S3DataType": "S3Prefix", + "CompressionType": "None", + }, + "HostingEulaKey": None, + }, + { + "ChannelName": "scripts", + "S3DataSource": { + "S3Uri": "s3://bucket/scripts/", + "S3DataType": "S3Prefix", + "CompressionType": "None", + }, + "HostingEulaKey": None, + }, + ] + mock_init.return_value = mock_init_kwargs + mock_create.return_value = Mock() + self.builder.mode = Mode.SAGEMAKER_ENDPOINT + self.builder.image_uri = None + + self.builder._build_for_jumpstart() + + self.assertEqual( + self.builder.additional_model_data_sources, + [ + { + "ChannelName": "draft_model", + "S3DataSource": { + "S3Uri": "s3://bucket/eagle/", + "S3DataType": "S3Prefix", + "CompressionType": "None", + }, + }, + { + "ChannelName": "scripts", + "S3DataSource": { + "S3Uri": "s3://bucket/scripts/", + "S3DataType": "S3Prefix", + "CompressionType": "None", + }, + }, + ], + ) + + @patch("sagemaker.core.jumpstart.factory.utils.get_init_kwargs") + @patch.object(MockModelBuilderServers, "_prepare_for_mode") + @patch.object(MockModelBuilderServers, "_create_model") + def test_build_public_additional_model_data_source_ignores_accept_eula( + self, mock_create, mock_prepare, mock_init + ): + """accept_eula on an ungated additional source is ignored: no + ModelAccessConfig is added (mirrors the v2 helper, which only applies + access configs to gated sources).""" + mock_init_kwargs = Mock() + mock_init_kwargs.image_uri = "djl-inference:0.21.0" + mock_init_kwargs.env = {} + mock_init_kwargs.model_data = "s3://bucket/model.tar.gz" + mock_init_kwargs.additional_model_data_sources = [ + { + "ChannelName": "draft_model", + "S3DataSource": { + "S3Uri": "s3://bucket/eagle/", + "S3DataType": "S3Prefix", + "CompressionType": "None", + }, + "HostingEulaKey": None, + } + ] + mock_init.return_value = mock_init_kwargs + mock_create.return_value = Mock() + self.builder.mode = Mode.SAGEMAKER_ENDPOINT + self.builder.image_uri = None + self.builder.accept_eula = True + + self.builder._build_for_jumpstart() + + self.assertEqual( + self.builder.additional_model_data_sources, + [ + { + "ChannelName": "draft_model", + "S3DataSource": { + "S3Uri": "s3://bucket/eagle/", + "S3DataType": "S3Prefix", + "CompressionType": "None", + }, + } + ], + ) + + @patch("sagemaker.core.jumpstart.factory.utils.get_init_kwargs") + @patch.object(MockModelBuilderServers, "_prepare_for_mode") + @patch.object(MockModelBuilderServers, "_create_model") + def test_build_mixed_additional_model_data_sources_with_accept_eula( + self, mock_create, mock_prepare, mock_init + ): + """With a mix of ungated and gated sources and accept_eula=True, the + ungated source passes through untouched while the gated source gets + ModelAccessConfig folded into its S3DataSource.""" + mock_init_kwargs = Mock() + mock_init_kwargs.image_uri = "djl-inference:0.21.0" + mock_init_kwargs.env = {} + mock_init_kwargs.model_data = "s3://bucket/model.tar.gz" + mock_init_kwargs.additional_model_data_sources = [ + { + "ChannelName": "draft_model", + "S3DataSource": { + "S3Uri": "s3://bucket/eagle/", + "S3DataType": "S3Prefix", + "CompressionType": "None", + }, + "HostingEulaKey": None, + }, + { + "ChannelName": "gated_draft_model", + "S3DataSource": { + "S3Uri": "s3://bucket/gated/", + "S3DataType": "S3Prefix", + "CompressionType": "None", + }, + "HostingEulaKey": "fmhMetadata/eula/llama3Eula.txt", + }, + ] + mock_init.return_value = mock_init_kwargs + mock_create.return_value = Mock() + self.builder.mode = Mode.SAGEMAKER_ENDPOINT + self.builder.image_uri = None + self.builder.accept_eula = True + + self.builder._build_for_jumpstart() + + self.assertEqual( + self.builder.additional_model_data_sources, + [ + { + "ChannelName": "draft_model", + "S3DataSource": { + "S3Uri": "s3://bucket/eagle/", + "S3DataType": "S3Prefix", + "CompressionType": "None", + }, + }, + { + "ChannelName": "gated_draft_model", + "S3DataSource": { + "S3Uri": "s3://bucket/gated/", + "S3DataType": "S3Prefix", + "CompressionType": "None", + "ModelAccessConfig": {"AcceptEula": True}, + }, + }, + ], + ) + + @patch("sagemaker.core.jumpstart.factory.utils.get_init_kwargs") + @patch.object(MockModelBuilderServers, "_prepare_for_mode") + @patch.object(MockModelBuilderServers, "_create_model") + def test_build_gated_additional_model_data_source_accept_eula_false_raises( + self, mock_create, mock_prepare, mock_init + ): + """An explicit accept_eula=False on a gated additional source raises, + matching the v2 helper's rejection of a false acceptance.""" + mock_init_kwargs = Mock() + mock_init_kwargs.image_uri = "djl-inference:0.21.0" + mock_init_kwargs.env = {} + mock_init_kwargs.model_data = "s3://bucket/model.tar.gz" + mock_init_kwargs.additional_model_data_sources = [ + { + "ChannelName": "gated_draft_model", + "S3DataSource": { + "S3Uri": "s3://bucket/gated/", + "S3DataType": "S3Prefix", + "CompressionType": "None", + }, + "HostingEulaKey": "fmhMetadata/eula/llama3Eula.txt", + } + ] + mock_init.return_value = mock_init_kwargs + mock_create.return_value = Mock() + self.builder.mode = Mode.SAGEMAKER_ENDPOINT + self.builder.image_uri = None + self.builder.accept_eula = False + + with self.assertRaises(ValueError) as ctx: + self.builder._build_for_jumpstart() + self.assertIn("accept_eula must be set to True", str(ctx.exception)) + + @patch("sagemaker.core.jumpstart.factory.utils.get_init_kwargs") + @patch.object(MockModelBuilderServers, "_prepare_for_mode") + @patch.object(MockModelBuilderServers, "_create_model") + def test_build_additional_model_data_sources_from_raw_spec_json( + self, mock_create, mock_prepare, mock_init + ): + """End-to-end from a raw JumpStart spec JSON: the real + JumpStartModelSpecs parsing and the real factory shaping helper + (_add_additional_model_data_sources_to_kwargs) produce the API-shape + init kwargs, and _build_for_jumpstart translates them into the final + CreateModel shape (HostingEulaKey stripped, EULA acceptance folded + into the gated source's ModelAccessConfig).""" + from sagemaker.core.jumpstart.factory.utils import ( + _add_additional_model_data_sources_to_kwargs, + ) + from sagemaker.core.jumpstart.types import JumpStartModelSpecs + + # Raw spec JSON in the snake_case shape published to the JumpStart + # content bucket (hosting_additional_data_sources.speculative_decoding). + raw_spec_json = { + "model_id": "pytorch-ic-mobilenet-v2", + "hosting_additional_data_sources": { + "speculative_decoding": [ + { + "channel_name": "draft_model", + "artifact_version": "v1", + "s3_data_source": { + "compression_type": "None", + "s3_data_type": "S3Prefix", + "s3_uri": "key/to/draft/model/", + }, + "provider": {"name": "JumpStart", "classification": "ungated"}, + }, + { + "channel_name": "gated_draft_model", + "artifact_version": "v1", + "s3_data_source": { + "compression_type": "None", + "s3_data_type": "S3Prefix", + "s3_uri": "key/to/gated/draft/", + }, + "hosting_eula_key": "fmhMetadata/eula/llama3Eula.txt", + "provider": {"name": "JumpStart", "classification": "gated"}, + }, + ] + }, + } + + # Run the real factory shaping over the real parsed spec, exactly as + # get_init_kwargs does, to produce the API-shape init kwargs. + mock_init_kwargs = Mock() + mock_init_kwargs.specs = JumpStartModelSpecs(raw_spec_json) + mock_init_kwargs.region = "us-west-2" + mock_init_kwargs.additional_model_data_sources = None + mock_init_kwargs = _add_additional_model_data_sources_to_kwargs( + kwargs=mock_init_kwargs + ) + mock_init_kwargs.image_uri = "djl-inference:0.21.0" + mock_init_kwargs.env = {} + mock_init_kwargs.model_data = "s3://bucket/model.tar.gz" + mock_init.return_value = mock_init_kwargs + + # The factory shaping keeps HostingEulaKey (internal metadata) in the + # API-shape kwargs; that is what _build_for_jumpstart must translate. + self.assertEqual( + [s.get("HostingEulaKey") for s in mock_init_kwargs.additional_model_data_sources], + [None, "fmhMetadata/eula/llama3Eula.txt"], + ) + + mock_create.return_value = Mock() + self.builder.mode = Mode.SAGEMAKER_ENDPOINT + self.builder.image_uri = None + self.builder.accept_eula = True + + self.builder._build_for_jumpstart() + + self.assertEqual( + self.builder.additional_model_data_sources, + [ + { + "ChannelName": "draft_model", + "S3DataSource": { + "CompressionType": "None", + "S3DataType": "S3Prefix", + "S3Uri": "s3://jumpstart-cache-prod-us-west-2/key/to/draft/model/", + }, + }, + { + "ChannelName": "gated_draft_model", + "S3DataSource": { + "CompressionType": "None", + "S3DataType": "S3Prefix", + "S3Uri": ( + "s3://jumpstart-private-cache-prod-us-west-2/key/to/gated/draft/" + ), + "ModelAccessConfig": {"AcceptEula": True}, + }, + }, + ], + ) + class TestDeployWrappers(unittest.TestCase): """Test deploy wrapper methods.""" From 46038f949ef09efd2a63be7a86b32b2a30d7a55a Mon Sep 17 00:00:00 2001 From: Eli Davidson Date: Tue, 4 Aug 2026 14:28:35 +0000 Subject: [PATCH 3/5] test(serve): drive additional-source EULA tests through the user flow Replace the mock-harness additional_model_data_sources tests with user-flow tests that exercise the documented user contract: construct a real ModelBuilder, set accept_eula, and call build(). Only the spec fetch boundary (JumpStartModelsAccessor.get_model_specs / _get_manifest) and the AWS boundary (_prepare_for_mode / _create_model, mock session) are mocked, mirroring how master-v2 tested JumpStartModel. Raw spec JSON fixtures live in data/jumpstart_specs.json as a base spec plus overlay variants (public model without/with public sources, gated sources, mixed), reusable across tests. Everything between the two boundaries runs for real: JumpStart detection, JumpStartModelSpecs parsing, the get_init_kwargs factory shaping, and the EULA translation in _build_for_jumpstart. --- .../unit/servers/data/jumpstart_specs.json | 898 ++++++++++++++++++ .../servers/test_model_builder_servers.py | 579 ++++------- 2 files changed, 1060 insertions(+), 417 deletions(-) create mode 100644 sagemaker-serve/tests/unit/servers/data/jumpstart_specs.json diff --git a/sagemaker-serve/tests/unit/servers/data/jumpstart_specs.json b/sagemaker-serve/tests/unit/servers/data/jumpstart_specs.json new file mode 100644 index 0000000000..dfedcde98e --- /dev/null +++ b/sagemaker-serve/tests/unit/servers/data/jumpstart_specs.json @@ -0,0 +1,898 @@ +{ + "_comment": "Raw JumpStart spec JSON fixtures for user-flow tests. base_spec is the prototypical pytorch-ic-mobilenet-v2 spec (same prototype the pre-v3 test suite used); each variant is an overlay merged over base_spec by the test loader. The shapes mirror what is published to the JumpStart content bucket (snake_case, hosting_additional_data_sources.speculative_decoding).", + "base_spec": { + "model_id": "pytorch-ic-mobilenet-v2", + "url": "https://pytorch.org/hub/pytorch_vision_mobilenet_v2/", + "version": "3.0.6", + "min_sdk_version": "2.189.0", + "training_supported": true, + "incremental_training_supported": true, + "hosting_ecr_specs": { + "framework": "pytorch", + "framework_version": "1.10.0", + "py_version": "py38" + }, + "hosting_artifact_key": "pytorch-ic/pytorch-ic-mobilenet-v2/artifacts/inference/v2.0.0/", + "hosting_script_key": "source-directory-tarballs/pytorch/inference/ic/v2.0.0/sourcedir.tar.gz", + "hosting_prepacked_artifact_key": "pytorch-ic/pytorch-ic-mobilenet-v2/artifacts/inference-prepack/v1.0.0/", + "hosting_prepacked_artifact_version": "1.0.0", + "hosting_use_script_uri": false, + "inference_vulnerable": false, + "inference_dependencies": [], + "inference_vulnerabilities": [], + "training_vulnerable": false, + "training_dependencies": [ + "sagemaker_jumpstart_prepack_script_utilities==1.0.0" + ], + "training_vulnerabilities": [], + "deprecated": false, + "hyperparameters": [ + { + "name": "train_only_top_layer", + "type": "text", + "default": "True", + "options": [ + "True", + "False" + ], + "scope": "algorithm" + }, + { + "name": "epochs", + "type": "int", + "default": 5, + "min": 1, + "max": 1000, + "scope": "algorithm" + }, + { + "name": "learning_rate", + "type": "float", + "default": 0.001, + "min": 1e-08, + "max": 1, + "scope": "algorithm" + }, + { + "name": "batch_size", + "type": "int", + "default": 4, + "min": 1, + "max": 1024, + "scope": "algorithm" + }, + { + "name": "reinitialize_top_layer", + "type": "text", + "default": "Auto", + "options": [ + "Auto", + "True", + "False" + ], + "scope": "algorithm" + }, + { + "name": "sagemaker_submit_directory", + "type": "text", + "default": "/opt/ml/input/data/code/sourcedir.tar.gz", + "scope": "container" + }, + { + "name": "sagemaker_program", + "type": "text", + "default": "transfer_learning.py", + "scope": "container" + }, + { + "name": "sagemaker_container_log_level", + "type": "text", + "default": "20", + "scope": "container" + } + ], + "training_script_key": "source-directory-tarballs/pytorch/transfer_learning/ic/v2.3.0/sourcedir.tar.gz", + "training_prepacked_script_key": "source-directory-tarballs/pytorch/transfer_learning/ic/prepack/v1.1.0/sourcedir.tar.gz", + "training_prepacked_script_version": "1.1.0", + "training_ecr_specs": { + "framework": "pytorch", + "framework_version": "1.10.0", + "py_version": "py38" + }, + "training_artifact_key": "pytorch-training/v2.0.0/train-pytorch-ic-mobilenet-v2.tar.gz", + "inference_environment_variables": [ + { + "name": "SAGEMAKER_PROGRAM", + "type": "text", + "default": "inference.py", + "scope": "container", + "required_for_model_class": true + }, + { + "name": "SAGEMAKER_SUBMIT_DIRECTORY", + "type": "text", + "default": "/opt/ml/model/code", + "scope": "container", + "required_for_model_class": false + }, + { + "name": "SAGEMAKER_CONTAINER_LOG_LEVEL", + "type": "text", + "default": "20", + "scope": "container", + "required_for_model_class": false + }, + { + "name": "SAGEMAKER_MODEL_SERVER_TIMEOUT", + "type": "text", + "default": "3600", + "scope": "container", + "required_for_model_class": false + }, + { + "name": "ENDPOINT_SERVER_TIMEOUT", + "type": "int", + "default": 3600, + "scope": "container", + "required_for_model_class": true + }, + { + "name": "MODEL_CACHE_ROOT", + "type": "text", + "default": "/opt/ml/model", + "scope": "container", + "required_for_model_class": true + }, + { + "name": "SAGEMAKER_ENV", + "type": "text", + "default": "1", + "scope": "container", + "required_for_model_class": true + }, + { + "name": "SAGEMAKER_MODEL_SERVER_WORKERS", + "type": "int", + "default": 1, + "scope": "container", + "required_for_model_class": true + } + ], + "metrics": [ + { + "Name": "pytorch-ic:val-accuracy", + "Regex": "val_accuracy: ([0-9\\.]+)" + } + ], + "default_inference_instance_type": "ml.m5.large", + "supported_inference_instance_types": [ + "ml.m5.large", + "ml.m5.xlarge", + "ml.c5.xlarge", + "ml.c5.2xlarge", + "ml.m4.large", + "ml.m4.xlarge" + ], + "default_training_instance_type": "ml.m5.xlarge", + "supported_training_instance_types": [ + "ml.m5.xlarge", + "ml.c5.2xlarge", + "ml.m4.xlarge" + ], + "model_kwargs": {}, + "deploy_kwargs": {}, + "estimator_kwargs": { + "encrypt_inter_container_traffic": true, + "max_run": 360000 + }, + "fit_kwargs": {}, + "predictor_specs": { + "supported_content_types": [ + "application/x-image" + ], + "supported_accept_types": [ + "application/json;verbose", + "application/json" + ], + "default_content_type": "application/x-image", + "default_accept_type": "application/json" + }, + "inference_enable_network_isolation": true, + "training_enable_network_isolation": true, + "default_training_dataset_key": "training-datasets/tf_flowers/", + "validation_supported": false, + "fine_tuning_supported": true, + "resource_name_base": "pt-ic-mobilenet-v2", + "hosting_instance_type_variants": { + "regional_aliases": { + "af-south-1": { + "cpu_ecr_uri_1": "626614931356.dkr.ecr.af-south-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "626614931356.dkr.ecr.af-south-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-east-1": { + "cpu_ecr_uri_1": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-northeast-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-northeast-2": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-northeast-3": { + "cpu_ecr_uri_1": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-south-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-southeast-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-southeast-2": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-southeast-3": { + "cpu_ecr_uri_1": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-southeast-5": { + "cpu_ecr_uri_1": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ca-central-1": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "cn-north-1": { + "cpu_ecr_uri_1": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/pytorch-inference:1.10.0-gpu-py38" + }, + "cn-northwest-1": { + "cpu_ecr_uri_1": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/pytorch-inference:1.10.0-gpu-py38" + }, + "eu-central-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "eu-central-2": { + "cpu_ecr_uri_1": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "eu-north-1": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "eu-south-1": { + "cpu_ecr_uri_1": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "eu-west-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "eu-west-2": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "eu-west-3": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "il-central-1": { + "cpu_ecr_uri_1": "780543022126.dkr.ecr.il-central-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "780543022126.dkr.ecr.il-central-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "me-central-1": { + "cpu_ecr_uri_1": "914824155844.dkr.ecr.me-central-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "914824155844.dkr.ecr.me-central-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "me-south-1": { + "cpu_ecr_uri_1": "217643126080.dkr.ecr.me-south-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "217643126080.dkr.ecr.me-south-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "sa-east-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "us-east-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "us-east-2": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "us-gov-east-1": { + "cpu_ecr_uri_1": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "us-gov-west-1": { + "cpu_ecr_uri_1": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "us-west-1": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-west-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-west-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "us-west-2": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + } + }, + "variants": { + "c4": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c5": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c5d": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c5n": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c6i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c6id": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c7i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "g4dn": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g5": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g6": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g6e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "local": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "local_gpu": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "m4": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m5": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m5d": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m6i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m6id": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m7i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "p2": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p3": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p3dn": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p4d": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p4de": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p5": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "r5": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r5d": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r6i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r6id": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r7i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "t2": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "t3": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "trn1": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_3" + } + }, + "trn1n": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_3" + } + } + } + }, + "training_instance_type_variants": { + "regional_aliases": { + "af-south-1": { + "cpu_ecr_uri_1": "626614931356.dkr.ecr.af-south-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "626614931356.dkr.ecr.af-south-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-east-1": { + "cpu_ecr_uri_1": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-northeast-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-northeast-2": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-northeast-3": { + "cpu_ecr_uri_1": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-south-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-southeast-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-southeast-2": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-southeast-3": { + "cpu_ecr_uri_1": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-southeast-5": { + "cpu_ecr_uri_1": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ca-central-1": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "cn-north-1": { + "cpu_ecr_uri_1": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/pytorch-training:1.10.0-gpu-py38" + }, + "cn-northwest-1": { + "cpu_ecr_uri_1": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/pytorch-training:1.10.0-gpu-py38" + }, + "eu-central-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "eu-central-2": { + "cpu_ecr_uri_1": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "eu-north-1": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "eu-south-1": { + "cpu_ecr_uri_1": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "eu-west-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "eu-west-2": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "eu-west-3": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "il-central-1": { + "cpu_ecr_uri_1": "780543022126.dkr.ecr.il-central-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "780543022126.dkr.ecr.il-central-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "me-central-1": { + "cpu_ecr_uri_1": "914824155844.dkr.ecr.me-central-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "914824155844.dkr.ecr.me-central-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "me-south-1": { + "cpu_ecr_uri_1": "217643126080.dkr.ecr.me-south-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "217643126080.dkr.ecr.me-south-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "sa-east-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "us-east-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "us-east-2": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "us-gov-east-1": { + "cpu_ecr_uri_1": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "us-gov-west-1": { + "cpu_ecr_uri_1": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "us-west-1": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-west-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-west-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "us-west-2": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + } + }, + "variants": { + "c4": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c5": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c5d": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c5n": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c6i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c6id": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c7i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "g4dn": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g5": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g6": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g6e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "local": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "local_gpu": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "m4": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m5": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m5d": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m6i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m6id": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m7i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "p2": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p3": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p3dn": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p4d": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p4de": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p5": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "r5": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r5d": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r6i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r6id": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r7i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "t2": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "t3": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "trn1": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_3" + } + }, + "trn1n": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_3" + } + } + } + }, + "hosting_artifact_s3_data_type": "S3Prefix", + "hosting_artifact_compression_type": "None", + "dynamic_container_deployment_supported": false + }, + "variants": { + "public_model_no_additional_sources": {}, + "public_model_public_additional_sources": { + "hosting_additional_data_sources": { + "speculative_decoding": [ + { + "channel_name": "draft_model", + "artifact_version": "v1", + "s3_data_source": { + "compression_type": "None", + "s3_data_type": "S3Prefix", + "s3_uri": "key/to/draft/model/" + }, + "provider": { + "name": "JumpStart", + "classification": "ungated" + } + } + ] + } + }, + "gated_model_gated_additional_sources": { + "hosting_eula_key": "fmhMetadata/eula/gatedModelEula.txt", + "hosting_additional_data_sources": { + "speculative_decoding": [ + { + "channel_name": "gated_draft_model", + "artifact_version": "v1", + "s3_data_source": { + "compression_type": "None", + "s3_data_type": "S3Prefix", + "s3_uri": "key/to/gated/draft/" + }, + "hosting_eula_key": "fmhMetadata/eula/gatedDraftModelEula.txt", + "provider": { + "name": "JumpStart", + "classification": "gated" + } + } + ] + } + }, + "public_model_mixed_additional_sources": { + "hosting_additional_data_sources": { + "speculative_decoding": [ + { + "channel_name": "draft_model", + "artifact_version": "v1", + "s3_data_source": { + "compression_type": "None", + "s3_data_type": "S3Prefix", + "s3_uri": "key/to/draft/model/" + }, + "provider": { + "name": "JumpStart", + "classification": "ungated" + } + }, + { + "channel_name": "gated_draft_model", + "artifact_version": "v1", + "s3_data_source": { + "compression_type": "None", + "s3_data_type": "S3Prefix", + "s3_uri": "key/to/gated/draft/" + }, + "hosting_eula_key": "fmhMetadata/eula/gatedDraftModelEula.txt", + "provider": { + "name": "JumpStart", + "classification": "gated" + } + } + ] + } + } + } +} \ No newline at end of file diff --git a/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py b/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py index 03f8c908c4..8a6471133f 100644 --- a/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py +++ b/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py @@ -1,5 +1,6 @@ """Unit tests for ModelBuilderServers class.""" +import json import os import sys import tempfile @@ -1004,460 +1005,204 @@ def test_build_sagemaker_endpoint_djl(self, mock_create, mock_prepare, mock_init mock_create.assert_called_once() - @patch("sagemaker.core.jumpstart.factory.utils.get_init_kwargs") - @patch.object(MockModelBuilderServers, "_prepare_for_mode") - @patch.object(MockModelBuilderServers, "_create_model") - def test_build_propagates_additional_model_data_sources( - self, mock_create, mock_prepare, mock_init - ): - """additional_model_data_sources from the JumpStart spec are propagated - to the builder with the JumpStart-internal HostingEulaKey metadata - stripped (mirroring _add_model_access_configs_to_model_data_sources).""" - mock_init_kwargs = Mock() - mock_init_kwargs.image_uri = "djl-inference:0.21.0" - mock_init_kwargs.env = {} - mock_init_kwargs.model_data = "s3://bucket/model.tar.gz" - # JumpStart spec includes an internal metadata key (HostingEulaKey) that - # the CreateModel API does not accept; it must be stripped. - mock_init_kwargs.additional_model_data_sources = [ - { - "ChannelName": "draft_model", - "S3DataSource": { - "S3Uri": "s3://bucket/eagle/", - "S3DataType": "S3Prefix", - "CompressionType": "None", - }, - "HostingEulaKey": None, - } - ] - mock_init.return_value = mock_init_kwargs - mock_create.return_value = Mock() - self.builder.mode = Mode.SAGEMAKER_ENDPOINT - self.builder.image_uri = None - - self.builder._build_for_jumpstart() - self.assertEqual( - self.builder.additional_model_data_sources, - [ - { - "ChannelName": "draft_model", - "S3DataSource": { - "S3Uri": "s3://bucket/eagle/", - "S3DataType": "S3Prefix", - "CompressionType": "None", - }, - } - ], +class TestJumpStartAdditionalModelDataSourcesUserFlow(unittest.TestCase): + """User-flow tests for additional model data sources, driven by raw spec JSON. + + Unlike the Mock-based tests above, these tests exercise the code the way a + user calls it: + + builder = ModelBuilder(model="", ...) + builder.accept_eula = True # the documented EULA knob + builder.build() + + Only the JSON-fetch boundary is mocked (the spec and manifest accessors, + fed from tests/unit/servers/data/jumpstart_specs.json) plus the AWS + boundary (_prepare_for_mode / _create_model). Everything in between -- + JumpStartModelSpecs parsing, the factory shaping in get_init_kwargs + (bucket resolution, PascalCase conversion), JumpStart model-id detection, + and _build_for_jumpstart's EULA handling -- runs for real. + + The fixture file holds one raw snake_case spec per use case (the shape + published to the JumpStart content bucket): public model without + additional sources, public model with a public source, gated model with a + gated source, and a public model with mixed public+gated sources. + """ + + SPEC_FIXTURE_PATH = Path(__file__).parent / "data" / "jumpstart_specs.json" + MODEL_ID = "pytorch-ic-mobilenet-v2" + + EXPECTED_PUBLIC_SOURCE = { + "ChannelName": "draft_model", + "S3DataSource": { + "CompressionType": "None", + "S3DataType": "S3Prefix", + "S3Uri": "s3://jumpstart-cache-prod-us-west-2/key/to/draft/model/", + }, + } + EXPECTED_GATED_SOURCE_ACCEPTED = { + "ChannelName": "gated_draft_model", + "S3DataSource": { + "CompressionType": "None", + "S3DataType": "S3Prefix", + "S3Uri": "s3://jumpstart-private-cache-prod-us-west-2/key/to/gated/draft/", + "ModelAccessConfig": {"AcceptEula": True}, + }, + } + + @classmethod + def setUpClass(cls): + with open(cls.SPEC_FIXTURE_PATH) as f: + cls.spec_fixture = json.load(f) + + def _spec_json(self, variant): + """Returns the raw spec JSON for a named fixture variant.""" + spec = json.loads(json.dumps(self.spec_fixture["base_spec"])) + spec.update(self.spec_fixture["variants"][variant]) + return spec + + def _make_builder(self, variant, accept_eula=None): + """Builds a real ModelBuilder the way a user would, with the JSON-fetch + boundary patched to serve the fixture variant.""" + from sagemaker.core.jumpstart.accessors import JumpStartModelsAccessor + from sagemaker.core.jumpstart.types import JumpStartModelHeader, JumpStartModelSpecs + from sagemaker.serve.model_builder import ModelBuilder + + spec_json = self._spec_json(variant) + + session = Mock() + session.boto_region_name = "us-west-2" + session.get_caller_identity_arn = Mock( + return_value="arn:aws:iam::123456789012:role/SageMakerRole" ) + session.sagemaker_config = {} + session.config = None - @patch("sagemaker.core.jumpstart.factory.utils.get_init_kwargs") - @patch.object(MockModelBuilderServers, "_prepare_for_mode") - @patch.object(MockModelBuilderServers, "_create_model") - def test_build_without_additional_model_data_sources( - self, mock_create, mock_prepare, mock_init - ): - """Models whose spec has no additional_model_data_sources are unaffected.""" - mock_init_kwargs = Mock() - mock_init_kwargs.image_uri = "djl-inference:0.21.0" - mock_init_kwargs.env = {} - mock_init_kwargs.model_data = "s3://bucket/model.tar.gz" - mock_init_kwargs.additional_model_data_sources = None - mock_init.return_value = mock_init_kwargs - mock_create.return_value = Mock() - self.builder.mode = Mode.SAGEMAKER_ENDPOINT - self.builder.image_uri = None - - self.builder._build_for_jumpstart() - - self.assertIsNone( - getattr(self.builder, "additional_model_data_sources", None) - ) - - @patch("sagemaker.core.jumpstart.factory.utils.get_init_kwargs") - @patch.object(MockModelBuilderServers, "_prepare_for_mode") - @patch.object(MockModelBuilderServers, "_create_model") - def test_build_gated_additional_model_data_source_with_accept_eula( - self, mock_create, mock_prepare, mock_init - ): - """A gated additional source (truthy HostingEulaKey) has the acceptance - folded into its ModelAccessConfig when accept_eula is set, mirroring the - base model's EULA handling.""" - mock_init_kwargs = Mock() - mock_init_kwargs.image_uri = "djl-inference:0.21.0" - mock_init_kwargs.env = {} - mock_init_kwargs.model_data = "s3://bucket/model.tar.gz" - mock_init_kwargs.additional_model_data_sources = [ + header = JumpStartModelHeader( { - "ChannelName": "draft_model", - "S3DataSource": { - "S3Uri": "s3://bucket/gated/", - "S3DataType": "S3Prefix", - "CompressionType": "None", - }, - "HostingEulaKey": "fmhMetadata/eula/llama3Eula.txt", + "model_id": self.MODEL_ID, + "version": "1.0.0", + "min_version": "2.49.0", + "spec_key": f"community_models_specs/{self.MODEL_ID}/specs_v1.0.0.json", } - ] - mock_init.return_value = mock_init_kwargs - mock_create.return_value = Mock() - self.builder.mode = Mode.SAGEMAKER_ENDPOINT - self.builder.image_uri = None - self.builder.accept_eula = True - - self.builder._build_for_jumpstart() - - self.assertEqual( - self.builder.additional_model_data_sources, - [ - { - "ChannelName": "draft_model", - "S3DataSource": { - "S3Uri": "s3://bucket/gated/", - "S3DataType": "S3Prefix", - "CompressionType": "None", - "ModelAccessConfig": {"AcceptEula": True}, - }, - } - ], ) - @patch("sagemaker.core.jumpstart.factory.utils.get_init_kwargs") - @patch.object(MockModelBuilderServers, "_prepare_for_mode") - @patch.object(MockModelBuilderServers, "_create_model") - def test_build_gated_additional_model_data_source_without_accept_eula_raises( - self, mock_create, mock_prepare, mock_init - ): - """A gated additional source without accept_eula raises, matching the - base model's EULA enforcement.""" - mock_init_kwargs = Mock() - mock_init_kwargs.image_uri = "djl-inference:0.21.0" - mock_init_kwargs.env = {} - mock_init_kwargs.model_data = "s3://bucket/model.tar.gz" - mock_init_kwargs.additional_model_data_sources = [ - { - "ChannelName": "draft_model", - "S3DataSource": { - "S3Uri": "s3://bucket/gated/", - "S3DataType": "S3Prefix", - "CompressionType": "None", - }, - "HostingEulaKey": "fmhMetadata/eula/llama3Eula.txt", - } + patchers = [ + patch.object( + JumpStartModelsAccessor, + "get_model_specs", + side_effect=lambda *a, **kw: JumpStartModelSpecs( + json.loads(json.dumps(spec_json)) + ), + ), + patch.object(JumpStartModelsAccessor, "_get_manifest", return_value=[header]), + patch.object(ModelBuilder, "_prepare_for_mode", return_value=None), + patch.object(ModelBuilder, "_create_model", return_value=Mock()), ] - mock_init.return_value = mock_init_kwargs - mock_create.return_value = Mock() - self.builder.mode = Mode.SAGEMAKER_ENDPOINT - self.builder.image_uri = None - self.builder.accept_eula = None + for p in patchers: + p.start() + self.addCleanup(p.stop) + + builder = ModelBuilder( + model=self.MODEL_ID, + mode=Mode.SAGEMAKER_ENDPOINT, + role_arn="arn:aws:iam::123456789012:role/SageMakerRole", + sagemaker_session=session, + ) + if accept_eula is not None: + builder.accept_eula = accept_eula + return builder - with self.assertRaises(ValueError) as ctx: - self.builder._build_for_jumpstart() - self.assertIn("accept_eula must be set to True", str(ctx.exception)) + def test_public_model_without_additional_sources(self): + """A spec with no additional data sources builds cleanly and sets none.""" + builder = self._make_builder("public_model_no_additional_sources") - @patch("sagemaker.core.jumpstart.factory.utils.get_init_kwargs") - @patch.object(MockModelBuilderServers, "_prepare_for_mode") - @patch.object(MockModelBuilderServers, "_create_model") - def test_build_multiple_public_additional_model_data_sources_pass_through( - self, mock_create, mock_prepare, mock_init - ): - """Multiple ungated additional sources are all propagated with only the - HostingEulaKey metadata stripped.""" - mock_init_kwargs = Mock() - mock_init_kwargs.image_uri = "djl-inference:0.21.0" - mock_init_kwargs.env = {} - mock_init_kwargs.model_data = "s3://bucket/model.tar.gz" - mock_init_kwargs.additional_model_data_sources = [ - { - "ChannelName": "draft_model", - "S3DataSource": { - "S3Uri": "s3://bucket/eagle/", - "S3DataType": "S3Prefix", - "CompressionType": "None", - }, - "HostingEulaKey": None, - }, - { - "ChannelName": "scripts", - "S3DataSource": { - "S3Uri": "s3://bucket/scripts/", - "S3DataType": "S3Prefix", - "CompressionType": "None", - }, - "HostingEulaKey": None, - }, - ] - mock_init.return_value = mock_init_kwargs - mock_create.return_value = Mock() - self.builder.mode = Mode.SAGEMAKER_ENDPOINT - self.builder.image_uri = None + builder.build() - self.builder._build_for_jumpstart() + self.assertIsNone(getattr(builder, "additional_model_data_sources", None)) + + def test_public_additional_source_builds_without_accept_eula(self): + """A public (ungated) additional source deploys without the user ever + touching accept_eula; the internal HostingEulaKey is stripped.""" + builder = self._make_builder("public_model_public_additional_sources") + + builder.build() self.assertEqual( - self.builder.additional_model_data_sources, - [ - { - "ChannelName": "draft_model", - "S3DataSource": { - "S3Uri": "s3://bucket/eagle/", - "S3DataType": "S3Prefix", - "CompressionType": "None", - }, - }, - { - "ChannelName": "scripts", - "S3DataSource": { - "S3Uri": "s3://bucket/scripts/", - "S3DataType": "S3Prefix", - "CompressionType": "None", - }, - }, - ], + builder.additional_model_data_sources, [self.EXPECTED_PUBLIC_SOURCE] ) - @patch("sagemaker.core.jumpstart.factory.utils.get_init_kwargs") - @patch.object(MockModelBuilderServers, "_prepare_for_mode") - @patch.object(MockModelBuilderServers, "_create_model") - def test_build_public_additional_model_data_source_ignores_accept_eula( - self, mock_create, mock_prepare, mock_init - ): - """accept_eula on an ungated additional source is ignored: no - ModelAccessConfig is added (mirrors the v2 helper, which only applies - access configs to gated sources).""" - mock_init_kwargs = Mock() - mock_init_kwargs.image_uri = "djl-inference:0.21.0" - mock_init_kwargs.env = {} - mock_init_kwargs.model_data = "s3://bucket/model.tar.gz" - mock_init_kwargs.additional_model_data_sources = [ - { - "ChannelName": "draft_model", - "S3DataSource": { - "S3Uri": "s3://bucket/eagle/", - "S3DataType": "S3Prefix", - "CompressionType": "None", - }, - "HostingEulaKey": None, - } - ] - mock_init.return_value = mock_init_kwargs - mock_create.return_value = Mock() - self.builder.mode = Mode.SAGEMAKER_ENDPOINT - self.builder.image_uri = None - self.builder.accept_eula = True + def test_public_additional_source_unaffected_by_accept_eula(self): + """Passing accept_eula=True for a public source is harmless: no + ModelAccessConfig is added.""" + builder = self._make_builder( + "public_model_public_additional_sources", accept_eula=True + ) - self.builder._build_for_jumpstart() + builder.build() self.assertEqual( - self.builder.additional_model_data_sources, - [ - { - "ChannelName": "draft_model", - "S3DataSource": { - "S3Uri": "s3://bucket/eagle/", - "S3DataType": "S3Prefix", - "CompressionType": "None", - }, - } - ], + builder.additional_model_data_sources, [self.EXPECTED_PUBLIC_SOURCE] ) - @patch("sagemaker.core.jumpstart.factory.utils.get_init_kwargs") - @patch.object(MockModelBuilderServers, "_prepare_for_mode") - @patch.object(MockModelBuilderServers, "_create_model") - def test_build_mixed_additional_model_data_sources_with_accept_eula( - self, mock_create, mock_prepare, mock_init - ): - """With a mix of ungated and gated sources and accept_eula=True, the - ungated source passes through untouched while the gated source gets - ModelAccessConfig folded into its S3DataSource.""" - mock_init_kwargs = Mock() - mock_init_kwargs.image_uri = "djl-inference:0.21.0" - mock_init_kwargs.env = {} - mock_init_kwargs.model_data = "s3://bucket/model.tar.gz" - mock_init_kwargs.additional_model_data_sources = [ - { - "ChannelName": "draft_model", - "S3DataSource": { - "S3Uri": "s3://bucket/eagle/", - "S3DataType": "S3Prefix", - "CompressionType": "None", - }, - "HostingEulaKey": None, - }, - { - "ChannelName": "gated_draft_model", - "S3DataSource": { - "S3Uri": "s3://bucket/gated/", - "S3DataType": "S3Prefix", - "CompressionType": "None", - }, - "HostingEulaKey": "fmhMetadata/eula/llama3Eula.txt", - }, - ] - mock_init.return_value = mock_init_kwargs - mock_create.return_value = Mock() - self.builder.mode = Mode.SAGEMAKER_ENDPOINT - self.builder.image_uri = None - self.builder.accept_eula = True + def test_gated_additional_source_with_accept_eula_builds(self): + """User sets accept_eula=True on the builder: the gated source deploys + with the acceptance recorded in its ModelAccessConfig.""" + builder = self._make_builder( + "gated_model_gated_additional_sources", accept_eula=True + ) - self.builder._build_for_jumpstart() + builder.build() self.assertEqual( - self.builder.additional_model_data_sources, - [ - { - "ChannelName": "draft_model", - "S3DataSource": { - "S3Uri": "s3://bucket/eagle/", - "S3DataType": "S3Prefix", - "CompressionType": "None", - }, - }, - { - "ChannelName": "gated_draft_model", - "S3DataSource": { - "S3Uri": "s3://bucket/gated/", - "S3DataType": "S3Prefix", - "CompressionType": "None", - "ModelAccessConfig": {"AcceptEula": True}, - }, - }, - ], + builder.additional_model_data_sources, [self.EXPECTED_GATED_SOURCE_ACCEPTED] ) - @patch("sagemaker.core.jumpstart.factory.utils.get_init_kwargs") - @patch.object(MockModelBuilderServers, "_prepare_for_mode") - @patch.object(MockModelBuilderServers, "_create_model") - def test_build_gated_additional_model_data_source_accept_eula_false_raises( - self, mock_create, mock_prepare, mock_init - ): - """An explicit accept_eula=False on a gated additional source raises, - matching the v2 helper's rejection of a false acceptance.""" - mock_init_kwargs = Mock() - mock_init_kwargs.image_uri = "djl-inference:0.21.0" - mock_init_kwargs.env = {} - mock_init_kwargs.model_data = "s3://bucket/model.tar.gz" - mock_init_kwargs.additional_model_data_sources = [ - { - "ChannelName": "gated_draft_model", - "S3DataSource": { - "S3Uri": "s3://bucket/gated/", - "S3DataType": "S3Prefix", - "CompressionType": "None", - }, - "HostingEulaKey": "fmhMetadata/eula/llama3Eula.txt", - } - ] - mock_init.return_value = mock_init_kwargs - mock_create.return_value = Mock() - self.builder.mode = Mode.SAGEMAKER_ENDPOINT - self.builder.image_uri = None - self.builder.accept_eula = False + def test_gated_additional_source_without_accept_eula_raises(self): + """User never sets accept_eula: build() fails with the actionable + EULA error instead of silently deploying the gated source.""" + builder = self._make_builder("gated_model_gated_additional_sources") with self.assertRaises(ValueError) as ctx: - self.builder._build_for_jumpstart() + builder.build() + self.assertIn("accept_eula must be set to True", str(ctx.exception)) - @patch("sagemaker.core.jumpstart.factory.utils.get_init_kwargs") - @patch.object(MockModelBuilderServers, "_prepare_for_mode") - @patch.object(MockModelBuilderServers, "_create_model") - def test_build_additional_model_data_sources_from_raw_spec_json( - self, mock_create, mock_prepare, mock_init - ): - """End-to-end from a raw JumpStart spec JSON: the real - JumpStartModelSpecs parsing and the real factory shaping helper - (_add_additional_model_data_sources_to_kwargs) produce the API-shape - init kwargs, and _build_for_jumpstart translates them into the final - CreateModel shape (HostingEulaKey stripped, EULA acceptance folded - into the gated source's ModelAccessConfig).""" - from sagemaker.core.jumpstart.factory.utils import ( - _add_additional_model_data_sources_to_kwargs, + def test_gated_additional_source_with_accept_eula_false_raises(self): + """User explicitly sets accept_eula=False: build() fails the same way.""" + builder = self._make_builder( + "gated_model_gated_additional_sources", accept_eula=False ) - from sagemaker.core.jumpstart.types import JumpStartModelSpecs - - # Raw spec JSON in the snake_case shape published to the JumpStart - # content bucket (hosting_additional_data_sources.speculative_decoding). - raw_spec_json = { - "model_id": "pytorch-ic-mobilenet-v2", - "hosting_additional_data_sources": { - "speculative_decoding": [ - { - "channel_name": "draft_model", - "artifact_version": "v1", - "s3_data_source": { - "compression_type": "None", - "s3_data_type": "S3Prefix", - "s3_uri": "key/to/draft/model/", - }, - "provider": {"name": "JumpStart", "classification": "ungated"}, - }, - { - "channel_name": "gated_draft_model", - "artifact_version": "v1", - "s3_data_source": { - "compression_type": "None", - "s3_data_type": "S3Prefix", - "s3_uri": "key/to/gated/draft/", - }, - "hosting_eula_key": "fmhMetadata/eula/llama3Eula.txt", - "provider": {"name": "JumpStart", "classification": "gated"}, - }, - ] - }, - } - - # Run the real factory shaping over the real parsed spec, exactly as - # get_init_kwargs does, to produce the API-shape init kwargs. - mock_init_kwargs = Mock() - mock_init_kwargs.specs = JumpStartModelSpecs(raw_spec_json) - mock_init_kwargs.region = "us-west-2" - mock_init_kwargs.additional_model_data_sources = None - mock_init_kwargs = _add_additional_model_data_sources_to_kwargs( - kwargs=mock_init_kwargs + + with self.assertRaises(ValueError) as ctx: + builder.build() + + self.assertIn("accept_eula must be set to True", str(ctx.exception)) + + def test_mixed_additional_sources_with_accept_eula(self): + """Public + gated sources together with accept_eula=True: the public + source passes through untouched, the gated source gets the + ModelAccessConfig.""" + builder = self._make_builder( + "public_model_mixed_additional_sources", accept_eula=True ) - mock_init_kwargs.image_uri = "djl-inference:0.21.0" - mock_init_kwargs.env = {} - mock_init_kwargs.model_data = "s3://bucket/model.tar.gz" - mock_init.return_value = mock_init_kwargs - # The factory shaping keeps HostingEulaKey (internal metadata) in the - # API-shape kwargs; that is what _build_for_jumpstart must translate. + builder.build() + self.assertEqual( - [s.get("HostingEulaKey") for s in mock_init_kwargs.additional_model_data_sources], - [None, "fmhMetadata/eula/llama3Eula.txt"], + builder.additional_model_data_sources, + [self.EXPECTED_PUBLIC_SOURCE, self.EXPECTED_GATED_SOURCE_ACCEPTED], ) - mock_create.return_value = Mock() - self.builder.mode = Mode.SAGEMAKER_ENDPOINT - self.builder.image_uri = None - self.builder.accept_eula = True + def test_mixed_additional_sources_without_accept_eula_raises(self): + """Public + gated sources together without accept_eula: the gated + source blocks the build.""" + builder = self._make_builder("public_model_mixed_additional_sources") - self.builder._build_for_jumpstart() + with self.assertRaises(ValueError) as ctx: + builder.build() - self.assertEqual( - self.builder.additional_model_data_sources, - [ - { - "ChannelName": "draft_model", - "S3DataSource": { - "CompressionType": "None", - "S3DataType": "S3Prefix", - "S3Uri": "s3://jumpstart-cache-prod-us-west-2/key/to/draft/model/", - }, - }, - { - "ChannelName": "gated_draft_model", - "S3DataSource": { - "CompressionType": "None", - "S3DataType": "S3Prefix", - "S3Uri": ( - "s3://jumpstart-private-cache-prod-us-west-2/key/to/gated/draft/" - ), - "ModelAccessConfig": {"AcceptEula": True}, - }, - }, - ], - ) + self.assertIn("accept_eula must be set to True", str(ctx.exception)) class TestDeployWrappers(unittest.TestCase): From bdb7de526b32fbcc679f956ba2174d15072b52fd Mon Sep 17 00:00:00 2001 From: Eli Davidson Date: Tue, 4 Aug 2026 14:59:02 +0000 Subject: [PATCH 4/5] test(serve): use real captured JumpStart specs as test fixtures Replace the synthetic base-spec-plus-variants fixture with real, unmodified specs captured from the production JumpStart content bucket, mirroring master-v2's PROTOTYPICAL_MODEL_SPECS_DICT pattern: the spec accessor routes by model id into one captured spec file per model, and each test calls the SDK exactly like a customer does. Fixture models (tests/unit/servers/data/jumpstart_specs/): - pytorch-ic-mobilenet-v2 v3.0.25: public, no additional data sources - openai-reasoning-gpt-oss-20b v3.38.0: ungated EAGLE speculative decoding source in its default (lmi-optimized) config -- the model that surfaced the propagation bug - meta-textgeneration-llama-3-1-70b v2.16.2: gated draft_model source (hosting_eula_key) in its lmi-optimized config, selected in tests via the public set_deployment_config API All patching is centralized in setUp (spec fetch, manifest, and the AWS boundary), so test bodies contain only public API calls: ModelBuilder(model=, instance_type=...), accept_eula, and build(). Real inference-config resolution now runs in every test. --- .../unit/servers/data/jumpstart_specs.json | 898 ---- .../data/jumpstart_specs/manifest.json | 38 + .../meta-textgeneration-llama-3-1-70b.json | 1910 +++++++++ .../openai-reasoning-gpt-oss-20b.json | 3801 +++++++++++++++++ .../pytorch-ic-mobilenet-v2.json | 1093 +++++ .../servers/test_model_builder_servers.py | 264 +- 6 files changed, 6990 insertions(+), 1014 deletions(-) delete mode 100644 sagemaker-serve/tests/unit/servers/data/jumpstart_specs.json create mode 100644 sagemaker-serve/tests/unit/servers/data/jumpstart_specs/manifest.json create mode 100644 sagemaker-serve/tests/unit/servers/data/jumpstart_specs/meta-textgeneration-llama-3-1-70b.json create mode 100644 sagemaker-serve/tests/unit/servers/data/jumpstart_specs/openai-reasoning-gpt-oss-20b.json create mode 100644 sagemaker-serve/tests/unit/servers/data/jumpstart_specs/pytorch-ic-mobilenet-v2.json diff --git a/sagemaker-serve/tests/unit/servers/data/jumpstart_specs.json b/sagemaker-serve/tests/unit/servers/data/jumpstart_specs.json deleted file mode 100644 index dfedcde98e..0000000000 --- a/sagemaker-serve/tests/unit/servers/data/jumpstart_specs.json +++ /dev/null @@ -1,898 +0,0 @@ -{ - "_comment": "Raw JumpStart spec JSON fixtures for user-flow tests. base_spec is the prototypical pytorch-ic-mobilenet-v2 spec (same prototype the pre-v3 test suite used); each variant is an overlay merged over base_spec by the test loader. The shapes mirror what is published to the JumpStart content bucket (snake_case, hosting_additional_data_sources.speculative_decoding).", - "base_spec": { - "model_id": "pytorch-ic-mobilenet-v2", - "url": "https://pytorch.org/hub/pytorch_vision_mobilenet_v2/", - "version": "3.0.6", - "min_sdk_version": "2.189.0", - "training_supported": true, - "incremental_training_supported": true, - "hosting_ecr_specs": { - "framework": "pytorch", - "framework_version": "1.10.0", - "py_version": "py38" - }, - "hosting_artifact_key": "pytorch-ic/pytorch-ic-mobilenet-v2/artifacts/inference/v2.0.0/", - "hosting_script_key": "source-directory-tarballs/pytorch/inference/ic/v2.0.0/sourcedir.tar.gz", - "hosting_prepacked_artifact_key": "pytorch-ic/pytorch-ic-mobilenet-v2/artifacts/inference-prepack/v1.0.0/", - "hosting_prepacked_artifact_version": "1.0.0", - "hosting_use_script_uri": false, - "inference_vulnerable": false, - "inference_dependencies": [], - "inference_vulnerabilities": [], - "training_vulnerable": false, - "training_dependencies": [ - "sagemaker_jumpstart_prepack_script_utilities==1.0.0" - ], - "training_vulnerabilities": [], - "deprecated": false, - "hyperparameters": [ - { - "name": "train_only_top_layer", - "type": "text", - "default": "True", - "options": [ - "True", - "False" - ], - "scope": "algorithm" - }, - { - "name": "epochs", - "type": "int", - "default": 5, - "min": 1, - "max": 1000, - "scope": "algorithm" - }, - { - "name": "learning_rate", - "type": "float", - "default": 0.001, - "min": 1e-08, - "max": 1, - "scope": "algorithm" - }, - { - "name": "batch_size", - "type": "int", - "default": 4, - "min": 1, - "max": 1024, - "scope": "algorithm" - }, - { - "name": "reinitialize_top_layer", - "type": "text", - "default": "Auto", - "options": [ - "Auto", - "True", - "False" - ], - "scope": "algorithm" - }, - { - "name": "sagemaker_submit_directory", - "type": "text", - "default": "/opt/ml/input/data/code/sourcedir.tar.gz", - "scope": "container" - }, - { - "name": "sagemaker_program", - "type": "text", - "default": "transfer_learning.py", - "scope": "container" - }, - { - "name": "sagemaker_container_log_level", - "type": "text", - "default": "20", - "scope": "container" - } - ], - "training_script_key": "source-directory-tarballs/pytorch/transfer_learning/ic/v2.3.0/sourcedir.tar.gz", - "training_prepacked_script_key": "source-directory-tarballs/pytorch/transfer_learning/ic/prepack/v1.1.0/sourcedir.tar.gz", - "training_prepacked_script_version": "1.1.0", - "training_ecr_specs": { - "framework": "pytorch", - "framework_version": "1.10.0", - "py_version": "py38" - }, - "training_artifact_key": "pytorch-training/v2.0.0/train-pytorch-ic-mobilenet-v2.tar.gz", - "inference_environment_variables": [ - { - "name": "SAGEMAKER_PROGRAM", - "type": "text", - "default": "inference.py", - "scope": "container", - "required_for_model_class": true - }, - { - "name": "SAGEMAKER_SUBMIT_DIRECTORY", - "type": "text", - "default": "/opt/ml/model/code", - "scope": "container", - "required_for_model_class": false - }, - { - "name": "SAGEMAKER_CONTAINER_LOG_LEVEL", - "type": "text", - "default": "20", - "scope": "container", - "required_for_model_class": false - }, - { - "name": "SAGEMAKER_MODEL_SERVER_TIMEOUT", - "type": "text", - "default": "3600", - "scope": "container", - "required_for_model_class": false - }, - { - "name": "ENDPOINT_SERVER_TIMEOUT", - "type": "int", - "default": 3600, - "scope": "container", - "required_for_model_class": true - }, - { - "name": "MODEL_CACHE_ROOT", - "type": "text", - "default": "/opt/ml/model", - "scope": "container", - "required_for_model_class": true - }, - { - "name": "SAGEMAKER_ENV", - "type": "text", - "default": "1", - "scope": "container", - "required_for_model_class": true - }, - { - "name": "SAGEMAKER_MODEL_SERVER_WORKERS", - "type": "int", - "default": 1, - "scope": "container", - "required_for_model_class": true - } - ], - "metrics": [ - { - "Name": "pytorch-ic:val-accuracy", - "Regex": "val_accuracy: ([0-9\\.]+)" - } - ], - "default_inference_instance_type": "ml.m5.large", - "supported_inference_instance_types": [ - "ml.m5.large", - "ml.m5.xlarge", - "ml.c5.xlarge", - "ml.c5.2xlarge", - "ml.m4.large", - "ml.m4.xlarge" - ], - "default_training_instance_type": "ml.m5.xlarge", - "supported_training_instance_types": [ - "ml.m5.xlarge", - "ml.c5.2xlarge", - "ml.m4.xlarge" - ], - "model_kwargs": {}, - "deploy_kwargs": {}, - "estimator_kwargs": { - "encrypt_inter_container_traffic": true, - "max_run": 360000 - }, - "fit_kwargs": {}, - "predictor_specs": { - "supported_content_types": [ - "application/x-image" - ], - "supported_accept_types": [ - "application/json;verbose", - "application/json" - ], - "default_content_type": "application/x-image", - "default_accept_type": "application/json" - }, - "inference_enable_network_isolation": true, - "training_enable_network_isolation": true, - "default_training_dataset_key": "training-datasets/tf_flowers/", - "validation_supported": false, - "fine_tuning_supported": true, - "resource_name_base": "pt-ic-mobilenet-v2", - "hosting_instance_type_variants": { - "regional_aliases": { - "af-south-1": { - "cpu_ecr_uri_1": "626614931356.dkr.ecr.af-south-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "626614931356.dkr.ecr.af-south-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "ap-east-1": { - "cpu_ecr_uri_1": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "ap-northeast-1": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "ap-northeast-2": { - "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "ap-northeast-3": { - "cpu_ecr_uri_1": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "ap-south-1": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "ap-southeast-1": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "ap-southeast-2": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "ap-southeast-3": { - "cpu_ecr_uri_1": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "ap-southeast-5": { - "cpu_ecr_uri_1": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "ca-central-1": { - "cpu_ecr_uri_1": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "cn-north-1": { - "cpu_ecr_uri_1": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/pytorch-inference:1.10.0-gpu-py38" - }, - "cn-northwest-1": { - "cpu_ecr_uri_1": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/pytorch-inference:1.10.0-gpu-py38" - }, - "eu-central-1": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "eu-central-2": { - "cpu_ecr_uri_1": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "eu-north-1": { - "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "eu-south-1": { - "cpu_ecr_uri_1": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "eu-west-1": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "eu-west-2": { - "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "eu-west-3": { - "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "il-central-1": { - "cpu_ecr_uri_1": "780543022126.dkr.ecr.il-central-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "780543022126.dkr.ecr.il-central-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "me-central-1": { - "cpu_ecr_uri_1": "914824155844.dkr.ecr.me-central-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "914824155844.dkr.ecr.me-central-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "me-south-1": { - "cpu_ecr_uri_1": "217643126080.dkr.ecr.me-south-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "217643126080.dkr.ecr.me-south-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "sa-east-1": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "us-east-1": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "us-east-2": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "us-gov-east-1": { - "cpu_ecr_uri_1": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "us-gov-west-1": { - "cpu_ecr_uri_1": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "us-west-1": { - "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-west-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-west-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - }, - "us-west-2": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" - } - }, - "variants": { - "c4": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "c5": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "c5d": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "c5n": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "c6i": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "c6id": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "c7i": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "g4dn": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "g5": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "g6": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "g6e": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "local": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "local_gpu": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "m4": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "m5": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "m5d": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "m6i": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "m6id": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "m7i": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "p2": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "p3": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "p3dn": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "p4d": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "p4de": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "p5": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "r5": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "r5d": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "r6i": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "r6id": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "r7i": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "t2": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "t3": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "trn1": { - "regional_properties": { - "image_uri": "$alias_ecr_uri_3" - } - }, - "trn1n": { - "regional_properties": { - "image_uri": "$alias_ecr_uri_3" - } - } - } - }, - "training_instance_type_variants": { - "regional_aliases": { - "af-south-1": { - "cpu_ecr_uri_1": "626614931356.dkr.ecr.af-south-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "626614931356.dkr.ecr.af-south-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "ap-east-1": { - "cpu_ecr_uri_1": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "ap-northeast-1": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "ap-northeast-2": { - "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "ap-northeast-3": { - "cpu_ecr_uri_1": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "ap-south-1": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "ap-southeast-1": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "ap-southeast-2": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "ap-southeast-3": { - "cpu_ecr_uri_1": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "ap-southeast-5": { - "cpu_ecr_uri_1": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "ca-central-1": { - "cpu_ecr_uri_1": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "cn-north-1": { - "cpu_ecr_uri_1": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/pytorch-training:1.10.0-gpu-py38" - }, - "cn-northwest-1": { - "cpu_ecr_uri_1": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/pytorch-training:1.10.0-gpu-py38" - }, - "eu-central-1": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "eu-central-2": { - "cpu_ecr_uri_1": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "eu-north-1": { - "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "eu-south-1": { - "cpu_ecr_uri_1": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "eu-west-1": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "eu-west-2": { - "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "eu-west-3": { - "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "il-central-1": { - "cpu_ecr_uri_1": "780543022126.dkr.ecr.il-central-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "780543022126.dkr.ecr.il-central-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "me-central-1": { - "cpu_ecr_uri_1": "914824155844.dkr.ecr.me-central-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "914824155844.dkr.ecr.me-central-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "me-south-1": { - "cpu_ecr_uri_1": "217643126080.dkr.ecr.me-south-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "217643126080.dkr.ecr.me-south-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "sa-east-1": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "us-east-1": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "us-east-2": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "us-gov-east-1": { - "cpu_ecr_uri_1": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "us-gov-west-1": { - "cpu_ecr_uri_1": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "us-west-1": { - "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-west-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-west-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - }, - "us-west-2": { - "alias_ecr_uri_3": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", - "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", - "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" - } - }, - "variants": { - "c4": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "c5": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "c5d": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "c5n": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "c6i": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "c6id": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "c7i": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "g4dn": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "g5": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "g6": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "g6e": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "local": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "local_gpu": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "m4": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "m5": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "m5d": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "m6i": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "m6id": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "m7i": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "p2": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "p3": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "p3dn": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "p4d": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "p4de": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "p5": { - "regional_properties": { - "image_uri": "$gpu_ecr_uri_2" - } - }, - "r5": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "r5d": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "r6i": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "r6id": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "r7i": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "t2": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "t3": { - "regional_properties": { - "image_uri": "$cpu_ecr_uri_1" - } - }, - "trn1": { - "regional_properties": { - "image_uri": "$alias_ecr_uri_3" - } - }, - "trn1n": { - "regional_properties": { - "image_uri": "$alias_ecr_uri_3" - } - } - } - }, - "hosting_artifact_s3_data_type": "S3Prefix", - "hosting_artifact_compression_type": "None", - "dynamic_container_deployment_supported": false - }, - "variants": { - "public_model_no_additional_sources": {}, - "public_model_public_additional_sources": { - "hosting_additional_data_sources": { - "speculative_decoding": [ - { - "channel_name": "draft_model", - "artifact_version": "v1", - "s3_data_source": { - "compression_type": "None", - "s3_data_type": "S3Prefix", - "s3_uri": "key/to/draft/model/" - }, - "provider": { - "name": "JumpStart", - "classification": "ungated" - } - } - ] - } - }, - "gated_model_gated_additional_sources": { - "hosting_eula_key": "fmhMetadata/eula/gatedModelEula.txt", - "hosting_additional_data_sources": { - "speculative_decoding": [ - { - "channel_name": "gated_draft_model", - "artifact_version": "v1", - "s3_data_source": { - "compression_type": "None", - "s3_data_type": "S3Prefix", - "s3_uri": "key/to/gated/draft/" - }, - "hosting_eula_key": "fmhMetadata/eula/gatedDraftModelEula.txt", - "provider": { - "name": "JumpStart", - "classification": "gated" - } - } - ] - } - }, - "public_model_mixed_additional_sources": { - "hosting_additional_data_sources": { - "speculative_decoding": [ - { - "channel_name": "draft_model", - "artifact_version": "v1", - "s3_data_source": { - "compression_type": "None", - "s3_data_type": "S3Prefix", - "s3_uri": "key/to/draft/model/" - }, - "provider": { - "name": "JumpStart", - "classification": "ungated" - } - }, - { - "channel_name": "gated_draft_model", - "artifact_version": "v1", - "s3_data_source": { - "compression_type": "None", - "s3_data_type": "S3Prefix", - "s3_uri": "key/to/gated/draft/" - }, - "hosting_eula_key": "fmhMetadata/eula/gatedDraftModelEula.txt", - "provider": { - "name": "JumpStart", - "classification": "gated" - } - } - ] - } - } - } -} \ No newline at end of file diff --git a/sagemaker-serve/tests/unit/servers/data/jumpstart_specs/manifest.json b/sagemaker-serve/tests/unit/servers/data/jumpstart_specs/manifest.json new file mode 100644 index 0000000000..486ca44a27 --- /dev/null +++ b/sagemaker-serve/tests/unit/servers/data/jumpstart_specs/manifest.json @@ -0,0 +1,38 @@ +[ + { + "model_id": "pytorch-ic-mobilenet-v2", + "version": "3.0.25", + "min_version": "2.189.0", + "spec_key": "community_models/pytorch-ic-mobilenet-v2/specs_v3.0.25.json", + "deprecated": false, + "provider": "pytorch", + "search_keywords": [ + "MobileNet V2", + "Vision", + "ResNet 18" + ] + }, + { + "model_id": "openai-reasoning-gpt-oss-20b", + "version": "3.38.0", + "min_version": "2.244.2", + "spec_key": "community_models/openai-reasoning-gpt-oss-20b/specs_v3.38.0.json", + "deprecated": false, + "provider": "openai", + "search_keywords": [ + "Text Generation", + "Reasoning" + ] + }, + { + "model_id": "meta-textgeneration-llama-3-1-70b", + "version": "2.16.2", + "min_version": "2.232.1", + "spec_key": "community_models/meta-textgeneration-llama-3-1-70b/specs_v2.16.2.json", + "deprecated": false, + "provider": "meta", + "search_keywords": [ + "Text Generation" + ] + } +] \ No newline at end of file diff --git a/sagemaker-serve/tests/unit/servers/data/jumpstart_specs/meta-textgeneration-llama-3-1-70b.json b/sagemaker-serve/tests/unit/servers/data/jumpstart_specs/meta-textgeneration-llama-3-1-70b.json new file mode 100644 index 0000000000..3b53a2e449 --- /dev/null +++ b/sagemaker-serve/tests/unit/servers/data/jumpstart_specs/meta-textgeneration-llama-3-1-70b.json @@ -0,0 +1,1910 @@ +{ + "default_inference_instance_type": "ml.p4d.24xlarge", + "default_training_dataset_key": "training-datasets/sec_amazon/", + "default_training_instance_type": "ml.g5.48xlarge", + "deprecated": false, + "dynamic_container_deployment_supported": true, + "estimator_kwargs": { + "disable_output_compression": true, + "encrypt_inter_container_traffic": true, + "max_run": 360000 + }, + "fine_tuning_supported": true, + "fit_kwargs": {}, + "gated_bucket": true, + "hosting_artifact_compression_type": "None", + "hosting_artifact_key": "meta-textgeneration/meta-textgeneration-llama-3-1-70b/artifacts/inference/v2.0.0/", + "hosting_artifact_s3_data_type": "S3Prefix", + "hosting_ecr_specs": { + "framework": "djl-lmi", + "framework_version": "0.29.0", + "py_version": "py310" + }, + "hosting_eula_key": "fmhMetadata/eula/llama3_1Eula.txt", + "hosting_neuron_model_id": "meta-textgenerationneuron-llama-3-1-70b", + "hosting_neuron_model_version": "1.0.2", + "hosting_prepacked_artifact_key": "meta-textgeneration/meta-textgeneration-llama-3-1-70b/artifacts/inference-prepack/v2.0.0/", + "hosting_prepacked_artifact_version": "2.0.0", + "hosting_script_key": "source-directory-tarballs/meta/inference/textgeneration/v1.2.3/sourcedir.tar.gz", + "hosting_use_script_uri": false, + "huggingface_id": "meta-llama/Llama-3.1-70B", + "hyperparameters": [ + { + "default": 5, + "max": 1000, + "min": 1, + "name": "epoch", + "scope": "algorithm", + "type": "int" + }, + { + "default": 0.0001, + "max": 1, + "min": 1e-08, + "name": "learning_rate", + "scope": "algorithm", + "type": "float" + }, + { + "default": 8, + "min": 1, + "name": "lora_r", + "scope": "algorithm", + "type": "int" + }, + { + "default": 32, + "min": 1, + "name": "lora_alpha", + "scope": "algorithm", + "type": "int" + }, + { + "default": "q_proj,v_proj", + "name": "target_modules", + "scope": "algorithm", + "type": "text" + }, + { + "default": 0.05, + "max": 1, + "min": 0, + "name": "lora_dropout", + "scope": "algorithm", + "type": "float" + }, + { + "default": "False", + "name": "instruction_tuned", + "options": [ + "True", + "False" + ], + "scope": "algorithm", + "type": "text" + }, + { + "default": "False", + "name": "chat_dataset", + "options": [ + "True", + "False" + ], + "scope": "algorithm", + "type": "text" + }, + { + "default": "True", + "name": "add_input_output_demarcation_key", + "options": [ + "True", + "False" + ], + "scope": "algorithm", + "type": "text" + }, + { + "default": 1, + "max": 1000, + "min": 1, + "name": "per_device_train_batch_size", + "scope": "algorithm", + "type": "int" + }, + { + "default": 1, + "max": 1000, + "min": 1, + "name": "per_device_eval_batch_size", + "scope": "algorithm", + "type": "int" + }, + { + "default": -1, + "min": -1, + "name": "max_train_samples", + "scope": "algorithm", + "type": "int" + }, + { + "default": -1, + "min": -1, + "name": "max_val_samples", + "scope": "algorithm", + "type": "int" + }, + { + "default": 10, + "max": 1000, + "min": 1, + "name": "seed", + "scope": "algorithm", + "type": "int" + }, + { + "default": -1, + "min": -1, + "name": "max_input_length", + "scope": "algorithm", + "type": "int" + }, + { + "default": 0.2, + "max": 1, + "min": 0, + "name": "validation_split_ratio", + "scope": "algorithm", + "type": "float" + }, + { + "default": 0, + "min": 0, + "name": "train_data_split_seed", + "scope": "algorithm", + "type": "int" + }, + { + "default": "None", + "name": "preprocessing_num_workers", + "scope": "algorithm", + "type": "text" + }, + { + "default": "True", + "name": "int8_quantization", + "options": [ + "True", + "False" + ], + "scope": "algorithm", + "type": "text" + }, + { + "default": "False", + "name": "enable_fsdp", + "options": [ + "True", + "False" + ], + "scope": "algorithm", + "type": "text" + }, + { + "default": "Llama3.1", + "name": "chat_template", + "scope": "algorithm", + "type": "text" + }, + { + "default": "/opt/ml/input/data/code/sourcedir.tar.gz", + "name": "sagemaker_submit_directory", + "scope": "container", + "type": "text" + }, + { + "default": "transfer_learning.py", + "name": "sagemaker_program", + "scope": "container", + "type": "text" + }, + { + "default": "20", + "name": "sagemaker_container_log_level", + "scope": "container", + "type": "text" + } + ], + "incremental_training_supported": false, + "inference_config_components": { + "lmi": { + "default_inference_instance_type": "ml.p4d.24xlarge", + "default_payloads": { + "englishToFrench": { + "body": { + "inputs": "Translate English to French:\nsea otter => loutre de mer\npeppermint => menthe poivr\u00e9e\nplush girafe => girafe peluche\ncheese =>", + "parameters": { + "max_new_tokens": 64, + "temperature": 0.6, + "top_p": 0.9 + } + }, + "content_type": "application/json", + "output_keys": { + "generated_text": "generated_text" + }, + "prompt_key": "inputs" + }, + "meaningOfLife": { + "body": { + "inputs": "I believe the meaning of life is", + "parameters": { + "details": true, + "max_new_tokens": 64, + "temperature": 0.6, + "top_p": 0.9 + } + }, + "content_type": "application/json", + "output_keys": { + "generated_text": "generated_text" + }, + "prompt_key": "inputs" + }, + "teamMessage": { + "body": { + "inputs": "A brief message congratulating the team on the launch:\n\nHi everyone,\n\nI just ", + "parameters": { + "max_new_tokens": 64, + "temperature": 0.6, + "top_p": 0.9 + } + }, + "content_type": "application/json", + "output_keys": { + "generated_text": "generated_text" + }, + "prompt_key": "inputs" + }, + "theoryOfRelativity": { + "body": { + "inputs": "Simply put, the theory of relativity states that ", + "parameters": { + "max_new_tokens": 64, + "temperature": 0.6, + "top_p": 0.9 + } + }, + "content_type": "application/json", + "output_keys": { + "generated_text": "generated_text" + }, + "prompt_key": "inputs" + } + }, + "deploy_kwargs": { + "container_startup_health_check_timeout": 1200, + "model_data_download_timeout": 1200 + }, + "hosting_artifact_compression_type": "None", + "hosting_artifact_key": "meta-textgeneration/meta-textgeneration-llama-3-1-70b/artifacts/inference/v2.0.0/", + "hosting_artifact_s3_data_type": "S3Prefix", + "hosting_ecr_specs": { + "framework": "djl-lmi", + "framework_version": "0.29.0", + "py_version": "py310" + }, + "hosting_instance_type_variants": { + "regional_aliases": { + "af-south-1": { + "alias_ecr_uri_1": "626614931356.dkr.ecr.af-south-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-east-1": { + "alias_ecr_uri_1": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-northeast-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-northeast-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-northeast-3": { + "alias_ecr_uri_1": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-south-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-south-2": { + "alias_ecr_uri_1": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-southeast-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-southeast-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-southeast-3": { + "alias_ecr_uri_1": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-southeast-4": { + "alias_ecr_uri_1": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-southeast-5": { + "alias_ecr_uri_1": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-southeast-7": { + "alias_ecr_uri_1": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ca-central-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ca-west-1": { + "alias_ecr_uri_1": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "cn-north-1": { + "alias_ecr_uri_1": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "cn-northwest-1": { + "alias_ecr_uri_1": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-central-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-central-2": { + "alias_ecr_uri_1": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-north-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-south-1": { + "alias_ecr_uri_1": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-south-2": { + "alias_ecr_uri_1": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-west-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-west-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-west-3": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "il-central-1": { + "alias_ecr_uri_1": "780543022126.dkr.ecr.il-central-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "me-central-1": { + "alias_ecr_uri_1": "914824155844.dkr.ecr.me-central-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "me-south-1": { + "alias_ecr_uri_1": "217643126080.dkr.ecr.me-south-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "mx-central-1": { + "alias_ecr_uri_1": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "sa-east-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-east-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-east-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-gov-east-1": { + "alias_ecr_uri_1": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-gov-west-1": { + "alias_ecr_uri_1": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-iso-east-1": { + "alias_ecr_uri_1": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-isob-east-1": { + "alias_ecr_uri_1": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-isof-east-1": { + "alias_ecr_uri_1": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-isof-south-1": { + "alias_ecr_uri_1": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-west-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-west-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-west-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + } + }, + "variants": { + "g4dn": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g5": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g6": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g6e": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g6f": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g7e": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "gr6": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "local_gpu": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "ml.g5.12xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "4" + } + } + }, + "ml.g5.24xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "4" + } + } + }, + "ml.g5.48xlarge": { + "properties": { + "environment_variables": { + "OPTION_GPU_MEMORY_UTILIZATION": "0.95", + "OPTION_TENSOR_PARALLEL_DEGREE": "8" + }, + "resource_requirements": { + "min_memory_mb": 393216, + "num_accelerators": 8 + } + } + }, + "ml.g6.12xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "4" + } + } + }, + "ml.g6.24xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "4" + } + } + }, + "ml.g6.48xlarge": { + "properties": { + "environment_variables": { + "OPTION_GPU_MEMORY_UTILIZATION": "0.95", + "OPTION_TENSOR_PARALLEL_DEGREE": "8" + }, + "resource_requirements": { + "min_memory_mb": 393216, + "num_accelerators": 8 + } + } + }, + "ml.p4d.24xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "8" + }, + "resource_requirements": { + "min_memory_mb": 589824, + "num_accelerators": 8 + } + } + }, + "ml.p4de.24xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "8" + } + } + }, + "ml.p5.48xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "4" + }, + "resource_requirements": { + "min_memory_mb": 1048576, + "num_accelerators": 8 + } + } + }, + "p3": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p3dn": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p4d": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p4de": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p5": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p5e": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p5en": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p6-b200": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p6-b300": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + } + } + }, + "hosting_neuron_model_id": "meta-textgenerationneuron-llama-3-1-70b", + "hosting_neuron_model_version": "1.0.2", + "hosting_prepacked_artifact_key": "meta-textgeneration/meta-textgeneration-llama-3-1-70b/artifacts/inference-prepack/v2.0.0/", + "hosting_prepacked_artifact_version": "2.0.0", + "hosting_resource_requirements": { + "min_memory_mb": 589824, + "num_accelerators": 8 + }, + "hosting_script_key": "source-directory-tarballs/meta/inference/textgeneration/v1.2.3/sourcedir.tar.gz", + "hosting_use_script_uri": false, + "inference_dependencies": [], + "inference_enable_network_isolation": true, + "inference_environment_variables": [ + { + "default": 3600, + "name": "ENDPOINT_SERVER_TIMEOUT", + "required_for_model_class": true, + "scope": "container", + "type": "int" + }, + { + "default": "/opt/ml/model", + "name": "HF_MODEL_ID", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model", + "name": "MODEL_CACHE_ROOT", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "true", + "name": "OPTION_ENFORCE_EAGER", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "20", + "name": "SAGEMAKER_CONTAINER_LOG_LEVEL", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "1", + "name": "SAGEMAKER_ENV", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "3600", + "name": "SAGEMAKER_MODEL_SERVER_TIMEOUT", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": 1, + "name": "SAGEMAKER_MODEL_SERVER_WORKERS", + "required_for_model_class": true, + "scope": "container", + "type": "int" + }, + { + "default": "inference.py", + "name": "SAGEMAKER_PROGRAM", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model/code", + "name": "SAGEMAKER_SUBMIT_DIRECTORY", + "required_for_model_class": true, + "scope": "container", + "type": "text" + } + ], + "inference_volume_size": 256, + "inference_vulnerabilities": [], + "inference_vulnerable": false, + "model_kwargs": {}, + "predictor_specs": { + "default_accept_type": "application/json", + "default_content_type": "application/json", + "supported_accept_types": [ + "application/json" + ], + "supported_content_types": [ + "application/json" + ] + }, + "supported_inference_instance_types": [ + "ml.g5.48xlarge", + "ml.p4d.24xlarge", + "ml.g6.48xlarge", + "ml.p5.48xlarge" + ] + }, + "lmi-optimized": { + "default_inference_instance_type": "ml.p4d.24xlarge", + "default_payloads": { + "englishToFrench": { + "body": { + "inputs": "Translate English to French:\nsea otter => loutre de mer\npeppermint => menthe poivr\u00e9e\nplush girafe => girafe peluche\ncheese =>", + "parameters": { + "max_new_tokens": 64, + "temperature": 0.6, + "top_p": 0.9 + } + }, + "content_type": "application/json", + "output_keys": { + "generated_text": "generated_text" + }, + "prompt_key": "inputs" + }, + "meaningOfLife": { + "body": { + "inputs": "I believe the meaning of life is", + "parameters": { + "details": true, + "max_new_tokens": 64, + "temperature": 0.6, + "top_p": 0.9 + } + }, + "content_type": "application/json", + "output_keys": { + "generated_text": "generated_text" + }, + "prompt_key": "inputs" + }, + "teamMessage": { + "body": { + "inputs": "A brief message congratulating the team on the launch:\n\nHi everyone,\n\nI just ", + "parameters": { + "max_new_tokens": 64, + "temperature": 0.6, + "top_p": 0.9 + } + }, + "content_type": "application/json", + "output_keys": { + "generated_text": "generated_text" + }, + "prompt_key": "inputs" + }, + "theoryOfRelativity": { + "body": { + "inputs": "Simply put, the theory of relativity states that ", + "parameters": { + "max_new_tokens": 64, + "temperature": 0.6, + "top_p": 0.9 + } + }, + "content_type": "application/json", + "output_keys": { + "generated_text": "generated_text" + }, + "prompt_key": "inputs" + } + }, + "deploy_kwargs": { + "container_startup_health_check_timeout": 1200, + "model_data_download_timeout": 1200 + }, + "hosting_additional_data_sources": { + "speculative_decoding": [ + { + "artifact_version": "v1.0.0", + "channel_name": "draft_model", + "hosting_eula_key": "fmhMetadata/eula/llama3_2Eula.txt", + "provider": { + "classification": "gated", + "name": "JumpStart" + }, + "s3_data_source": { + "compression_type": "None", + "s3_data_type": "S3Prefix", + "s3_uri": "meta-textgeneration/meta-textgeneration-llama-3-2-1b/artifacts/inference-prepack/v1.0.0/" + } + } + ] + }, + "hosting_artifact_compression_type": "None", + "hosting_artifact_key": "meta-textgeneration/meta-textgeneration-llama-3-1-70b/artifacts/inference/v2.0.0/", + "hosting_artifact_s3_data_type": "S3Prefix", + "hosting_ecr_specs": { + "framework": "djl-lmi", + "framework_version": "0.29.0", + "py_version": "py310" + }, + "hosting_instance_type_variants": { + "regional_aliases": { + "af-south-1": { + "alias_ecr_uri_1": "626614931356.dkr.ecr.af-south-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-east-1": { + "alias_ecr_uri_1": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-northeast-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-northeast-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-northeast-3": { + "alias_ecr_uri_1": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-south-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-south-2": { + "alias_ecr_uri_1": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-southeast-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-southeast-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-southeast-3": { + "alias_ecr_uri_1": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-southeast-4": { + "alias_ecr_uri_1": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-southeast-5": { + "alias_ecr_uri_1": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ap-southeast-7": { + "alias_ecr_uri_1": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ca-central-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "ca-west-1": { + "alias_ecr_uri_1": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "cn-north-1": { + "alias_ecr_uri_1": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "cn-northwest-1": { + "alias_ecr_uri_1": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-central-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-central-2": { + "alias_ecr_uri_1": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-north-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-south-1": { + "alias_ecr_uri_1": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-south-2": { + "alias_ecr_uri_1": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-west-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-west-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "eu-west-3": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "il-central-1": { + "alias_ecr_uri_1": "780543022126.dkr.ecr.il-central-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "me-central-1": { + "alias_ecr_uri_1": "914824155844.dkr.ecr.me-central-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "me-south-1": { + "alias_ecr_uri_1": "217643126080.dkr.ecr.me-south-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "mx-central-1": { + "alias_ecr_uri_1": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "sa-east-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-east-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-east-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-gov-east-1": { + "alias_ecr_uri_1": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-gov-west-1": { + "alias_ecr_uri_1": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-iso-east-1": { + "alias_ecr_uri_1": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-isob-east-1": { + "alias_ecr_uri_1": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-isof-east-1": { + "alias_ecr_uri_1": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-isof-south-1": { + "alias_ecr_uri_1": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-west-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-west-1.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + }, + "us-west-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.29.0-lmi11.0.0-cu124" + } + }, + "variants": { + "g4dn": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g5": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g6": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g6e": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g6f": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g7e": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "gr6": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "local_gpu": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "ml.g5.12xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "4" + } + } + }, + "ml.g5.24xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "4" + } + } + }, + "ml.g5.48xlarge": { + "properties": { + "environment_variables": { + "OPTION_GPU_MEMORY_UTILIZATION": "0.95", + "OPTION_TENSOR_PARALLEL_DEGREE": "8" + }, + "resource_requirements": { + "min_memory_mb": 393216, + "num_accelerators": 8 + } + } + }, + "ml.g6.12xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "4" + } + } + }, + "ml.g6.24xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "4" + } + } + }, + "ml.g6.48xlarge": { + "properties": { + "environment_variables": { + "OPTION_GPU_MEMORY_UTILIZATION": "0.95", + "OPTION_TENSOR_PARALLEL_DEGREE": "8" + }, + "resource_requirements": { + "min_memory_mb": 393216, + "num_accelerators": 8 + } + } + }, + "ml.p4d.24xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "8" + }, + "resource_requirements": { + "min_memory_mb": 589824, + "num_accelerators": 8 + } + } + }, + "ml.p4de.24xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "8" + } + } + }, + "ml.p5.48xlarge": { + "properties": { + "environment_variables": { + "OPTION_TENSOR_PARALLEL_DEGREE": "8" + }, + "resource_requirements": { + "min_memory_mb": 1048576, + "num_accelerators": 8 + } + } + }, + "p3": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p3dn": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p4d": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p4de": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p5": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p5e": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p5en": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p6-b200": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p6-b300": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + } + } + }, + "hosting_neuron_model_id": "meta-textgenerationneuron-llama-3-1-70b", + "hosting_neuron_model_version": "1.0.2", + "hosting_prepacked_artifact_key": "meta-textgeneration/meta-textgeneration-llama-3-1-70b/artifacts/inference-prepack/v2.0.0/", + "hosting_prepacked_artifact_version": "2.0.0", + "hosting_resource_requirements": { + "min_memory_mb": 589824, + "num_accelerators": 8 + }, + "hosting_script_key": "source-directory-tarballs/meta/inference/textgeneration/v1.2.3/sourcedir.tar.gz", + "hosting_use_script_uri": false, + "inference_dependencies": [], + "inference_enable_network_isolation": true, + "inference_environment_variables": [ + { + "default": 3600, + "name": "ENDPOINT_SERVER_TIMEOUT", + "required_for_model_class": true, + "scope": "container", + "type": "int" + }, + { + "default": "/opt/ml/model", + "name": "HF_MODEL_ID", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model", + "name": "MODEL_CACHE_ROOT", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/additional-model-data-sources/draft_model", + "name": "OPTION_SPECULATIVE_DRAFT_MODEL", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "20", + "name": "SAGEMAKER_CONTAINER_LOG_LEVEL", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "1", + "name": "SAGEMAKER_ENV", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "3600", + "name": "SAGEMAKER_MODEL_SERVER_TIMEOUT", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": 1, + "name": "SAGEMAKER_MODEL_SERVER_WORKERS", + "required_for_model_class": true, + "scope": "container", + "type": "int" + }, + { + "default": "inference.py", + "name": "SAGEMAKER_PROGRAM", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model/code", + "name": "SAGEMAKER_SUBMIT_DIRECTORY", + "required_for_model_class": true, + "scope": "container", + "type": "text" + } + ], + "inference_volume_size": 256, + "inference_vulnerabilities": [], + "inference_vulnerable": false, + "model_kwargs": {}, + "predictor_specs": { + "default_accept_type": "application/json", + "default_content_type": "application/json", + "supported_accept_types": [ + "application/json" + ], + "supported_content_types": [ + "application/json" + ] + }, + "supported_inference_instance_types": [ + "ml.g5.48xlarge", + "ml.g6.48xlarge", + "ml.p4d.24xlarge", + "ml.p5.48xlarge" + ] + } + }, + "inference_config_rankings": { + "overall": { + "description": "default", + "rankings": [ + "lmi", + "lmi-optimized" + ] + } + }, + "inference_configs": { + "lmi": { + "benchmark_metrics": { + "ml.g5.48xlarge": [ + { + "concurrency": "1", + "name": "latency", + "unit": "sec", + "value": "2.68" + }, + { + "concurrency": "1", + "name": "throughput", + "unit": "tokens/sec", + "value": "18.6" + } + ], + "ml.g6.48xlarge": [ + { + "concurrency": "8", + "name": "latency", + "unit": "sec", + "value": "1.41" + }, + { + "concurrency": "8", + "name": "throughput", + "unit": "tokens/sec", + "value": "4.9" + } + ], + "ml.p4d.24xlarge": [ + { + "concurrency": "4", + "name": "latency", + "unit": "sec", + "value": "0.11" + }, + { + "concurrency": "4", + "name": "throughput", + "unit": "tokens/sec", + "value": "37.4" + }, + { + "concurrency": "16", + "name": "latency", + "unit": "sec", + "value": "0.13" + }, + { + "concurrency": "16", + "name": "throughput", + "unit": "tokens/sec", + "value": "21.6" + }, + { + "concurrency": "32", + "name": "latency", + "unit": "sec", + "value": "0.16" + }, + { + "concurrency": "32", + "name": "throughput", + "unit": "tokens/sec", + "value": "13.9" + }, + { + "concurrency": "64", + "name": "latency", + "unit": "sec", + "value": "0.20" + }, + { + "concurrency": "64", + "name": "throughput", + "unit": "tokens/sec", + "value": "8.4" + }, + { + "concurrency": "128", + "name": "latency", + "unit": "sec", + "value": "3.91" + }, + { + "concurrency": "128", + "name": "throughput", + "unit": "tokens/sec", + "value": "7.6" + } + ], + "ml.p5.48xlarge": [ + { + "concurrency": "1", + "name": "latency", + "unit": "sec", + "value": "0.08" + }, + { + "concurrency": "1", + "name": "throughput", + "unit": "tokens/sec", + "value": "48.3" + }, + { + "concurrency": "2", + "name": "latency", + "unit": "sec", + "value": "0.08" + }, + { + "concurrency": "2", + "name": "throughput", + "unit": "tokens/sec", + "value": "48.3" + }, + { + "concurrency": "4", + "name": "latency", + "unit": "sec", + "value": "0.08" + }, + { + "concurrency": "4", + "name": "throughput", + "unit": "tokens/sec", + "value": "46.3" + }, + { + "concurrency": "8", + "name": "latency", + "unit": "sec", + "value": "0.08" + }, + { + "concurrency": "8", + "name": "throughput", + "unit": "tokens/sec", + "value": "42.1" + }, + { + "concurrency": "16", + "name": "latency", + "unit": "sec", + "value": "0.09" + }, + { + "concurrency": "16", + "name": "throughput", + "unit": "tokens/sec", + "value": "35.8" + }, + { + "concurrency": "32", + "name": "latency", + "unit": "sec", + "value": "0.09" + }, + { + "concurrency": "32", + "name": "throughput", + "unit": "tokens/sec", + "value": "26.6" + }, + { + "concurrency": "64", + "name": "latency", + "unit": "sec", + "value": "0.10" + }, + { + "concurrency": "64", + "name": "throughput", + "unit": "tokens/sec", + "value": "17.3" + } + ] + }, + "component_names": [ + "lmi" + ] + }, + "lmi-optimized": { + "acceleration_configs": [ + { + "diy_workflow_overrides": { + "gpu-lmi-trt": { + "enabled": false, + "reason": "TRT-LLM 0.11.0 in LMI v11 does not support llama 3.1" + } + }, + "enabled": false, + "type": "Compilation" + }, + { + "diy_workflow_overrides": { + "gpu-lmi-trt": { + "enabled": false, + "reason": "LMI v11 does not support Speculative Decoding for TRT" + } + }, + "enabled": true, + "type": "Speculative-Decoding" + }, + { + "diy_workflow_overrides": { + "gpu-lmi-trt": { + "enabled": false, + "reason": "TRT-LLM 0.11.0 in LMI v11 does not support llama 3.1" + } + }, + "enabled": false, + "type": "Quantization" + } + ], + "benchmark_metrics": { + "ml.g5.48xlarge": [ + { + "concurrency": "2", + "name": "latency", + "unit": "sec", + "value": "2.94" + }, + { + "concurrency": "2", + "name": "throughput", + "unit": "tokens/sec", + "value": "16.1" + }, + { + "concurrency": "4", + "name": "latency", + "unit": "sec", + "value": "4.04" + }, + { + "concurrency": "4", + "name": "throughput", + "unit": "tokens/sec", + "value": "6.1" + } + ], + "ml.g6.48xlarge": [ + { + "concurrency": "1", + "name": "latency", + "unit": "sec", + "value": "0.93" + }, + { + "concurrency": "1", + "name": "throughput", + "unit": "tokens/sec", + "value": "23.5" + }, + { + "concurrency": "2", + "name": "latency", + "unit": "sec", + "value": "1.14" + }, + { + "concurrency": "2", + "name": "throughput", + "unit": "tokens/sec", + "value": "17.7" + }, + { + "concurrency": "4", + "name": "latency", + "unit": "sec", + "value": "1.15" + }, + { + "concurrency": "4", + "name": "throughput", + "unit": "tokens/sec", + "value": "11.2" + } + ], + "ml.p4d.24xlarge": [ + { + "concurrency": "1", + "name": "latency", + "unit": "sec", + "value": "0.10" + }, + { + "concurrency": "1", + "name": "throughput", + "unit": "tokens/sec", + "value": "89.4" + }, + { + "concurrency": "2", + "name": "latency", + "unit": "sec", + "value": "0.13" + }, + { + "concurrency": "2", + "name": "throughput", + "unit": "tokens/sec", + "value": "81.2" + }, + { + "concurrency": "8", + "name": "latency", + "unit": "sec", + "value": "0.15" + }, + { + "concurrency": "8", + "name": "throughput", + "unit": "tokens/sec", + "value": "49.1" + } + ] + }, + "component_names": [ + "lmi-optimized" + ] + } + }, + "inference_dependencies": [], + "inference_enable_network_isolation": true, + "inference_environment_variables": [ + { + "default": 3600, + "name": "ENDPOINT_SERVER_TIMEOUT", + "required_for_model_class": true, + "scope": "container", + "type": "int" + }, + { + "default": "/opt/ml/model", + "name": "HF_MODEL_ID", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model", + "name": "MODEL_CACHE_ROOT", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "true", + "name": "OPTION_ENFORCE_EAGER", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "20", + "name": "SAGEMAKER_CONTAINER_LOG_LEVEL", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "1", + "name": "SAGEMAKER_ENV", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "3600", + "name": "SAGEMAKER_MODEL_SERVER_TIMEOUT", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": 1, + "name": "SAGEMAKER_MODEL_SERVER_WORKERS", + "required_for_model_class": true, + "scope": "container", + "type": "int" + }, + { + "default": "inference.py", + "name": "SAGEMAKER_PROGRAM", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model/code", + "name": "SAGEMAKER_SUBMIT_DIRECTORY", + "required_for_model_class": true, + "scope": "container", + "type": "text" + } + ], + "inference_volume_size": 256, + "inference_vulnerabilities": [], + "inference_vulnerable": false, + "input_modalities": [ + "Text" + ], + "languages": [ + "English", + "German", + "French", + "Italian", + "Portuguese", + "Hindi", + "Spanish", + "Thai" + ], + "license": "Llama 3.1 Community License", + "metrics": [ + { + "Name": "huggingface-textgeneration:eval-loss", + "Regex": "eval_epoch_loss=tensor\\(([0-9\\.]+)" + }, + { + "Name": "huggingface-textgeneration:eval-ppl", + "Regex": "eval_ppl=tensor\\(([0-9\\.]+)" + }, + { + "Name": "huggingface-textgeneration:train-loss", + "Regex": "train_epoch_loss=([0-9\\.]+)" + } + ], + "min_sdk_version": "2.232.1", + "ml_framework": "Meta", + "model_access": "Gated", + "model_id": "meta-textgeneration-llama-3-1-70b", + "model_kwargs": {}, + "model_size": "70B-100B", + "output_modalities": [ + "Text" + ], + "provider": "meta", + "resource_name_base": "meta-textgeneration-llama-3-1-70b", + "supported_inference_instance_types": [ + "ml.g5.48xlarge", + "ml.p4d.24xlarge", + "ml.g6.48xlarge", + "ml.p5.48xlarge" + ], + "supported_training_instance_types": [ + "ml.g5.48xlarge", + "ml.p4d.24xlarge" + ], + "training_artifact_key": "meta-training/train-meta-textgeneration-llama-3-1-70b.tar.gz", + "training_dependencies": [ + "accelerate==0.33.0", + "bitsandbytes==0.39.1", + "black==23.7.0", + "brotli==1.0.9", + "datasets==2.14.1", + "docstring-parser==0.16", + "fire==0.5.0", + "huggingface-hub==0.24.2", + "inflate64==0.3.1", + "loralib==0.1.1", + "multivolumefile==0.2.3", + "mypy-extensions==1.0.0", + "nvidia-cublas-cu12==12.1.3.1", + "nvidia-cuda-cupti-cu12==12.1.105", + "nvidia-cuda-nvrtc-cu12==12.1.105", + "nvidia-cuda-runtime-cu12==12.1.105", + "nvidia-cudnn-cu12==8.9.2.26", + "nvidia-cufft-cu12==11.0.2.54", + "nvidia-curand-cu12==10.3.2.106", + "nvidia-cusolver-cu12==11.4.5.107", + "nvidia-cusolver-cu12==11.4.5.107", + "nvidia-cusparse-cu12==12.1.0.106", + "nvidia-nccl-cu12==2.19.3", + "nvidia-nvjitlink-cu12==12.3.101", + "nvidia-nvtx-cu12==12.1.105", + "pathspec==0.11.1", + "peft==0.4.0", + "py7zr==0.20.5", + "pybcj==1.0.1", + "pycryptodomex==3.18.0", + "pyppmd==1.0.0", + "pyzstd==0.15.9", + "safetensors==0.4.2", + "sagemaker_jumpstart_huggingface_script_utilities==1.2.7", + "sagemaker_jumpstart_script_utilities==1.1.9", + "scipy==1.11.1", + "shtab==1.7.1", + "termcolor==2.3.0", + "texttable==1.6.7", + "tokenize-rt==5.1.0", + "tokenizers==0.19.1", + "torch==2.2.0", + "transformers==4.43.1", + "triton==2.2.0", + "trl==0.8.1", + "typing-extensions==4.8.0", + "tyro==0.7.3" + ], + "training_ecr_specs": { + "framework": "huggingface", + "framework_version": "2.0.0", + "huggingface_transformers_version": "4.28.1", + "py_version": "py310" + }, + "training_enable_network_isolation": true, + "training_instance_type_variants": { + "regional_aliases": { + "af-south-1": { + "gpu_ecr_uri_1": "626614931356.dkr.ecr.af-south-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "ap-east-1": { + "gpu_ecr_uri_1": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "ap-northeast-1": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "ap-northeast-2": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "ap-northeast-3": { + "gpu_ecr_uri_1": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "ap-south-1": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "ap-southeast-1": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "ap-southeast-2": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "ap-southeast-3": { + "gpu_ecr_uri_1": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "ca-central-1": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "ca-west-1": { + "gpu_ecr_uri_1": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "cn-north-1": { + "gpu_ecr_uri_1": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "cn-northwest-1": { + "gpu_ecr_uri_1": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "eu-central-1": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "eu-north-1": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "eu-south-1": { + "gpu_ecr_uri_1": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "eu-west-1": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "eu-west-2": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "eu-west-3": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "il-central-1": { + "gpu_ecr_uri_1": "780543022126.dkr.ecr.il-central-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "me-central-1": { + "gpu_ecr_uri_1": "914824155844.dkr.ecr.me-central-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "me-south-1": { + "gpu_ecr_uri_1": "217643126080.dkr.ecr.me-south-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "sa-east-1": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "us-east-1": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "us-east-2": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.us-east-2.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "us-gov-east-1": { + "gpu_ecr_uri_1": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "us-gov-west-1": { + "gpu_ecr_uri_1": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "us-iso-east-1": { + "gpu_ecr_uri_1": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "us-isob-east-1": { + "gpu_ecr_uri_1": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "us-west-1": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.us-west-1.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + }, + "us-west-2": { + "gpu_ecr_uri_1": "763104351884.dkr.ecr.us-west-2.amazonaws.com/huggingface-pytorch-training:2.0.0-transformers4.28.1-gpu-py310-cu118-ubuntu20.04" + } + }, + "variants": { + "g4dn": { + "properties": { + "gated_model_key_env_var_value": "meta-training/g4dn/v1.0.0/train-meta-textgeneration-llama-3-1-70b.tar.gz" + }, + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "g5": { + "properties": { + "gated_model_key_env_var_value": "meta-training/g5/v1.0.0/train-meta-textgeneration-llama-3-1-70b.tar.gz" + }, + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "g6": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "g6e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "g6f": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "g7e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "gr6": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "local_gpu": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "p3": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "p3dn": { + "properties": { + "gated_model_key_env_var_value": "meta-training/p3dn/v1.0.0/train-meta-textgeneration-llama-3-1-70b.tar.gz" + }, + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "p4d": { + "properties": { + "gated_model_key_env_var_value": "meta-training/p4d/v1.0.0/train-meta-textgeneration-llama-3-1-70b.tar.gz" + }, + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "p4de": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "p5": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "p5e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "p5en": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "p6-b200": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + }, + "p6-b300": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_1" + } + } + } + }, + "training_prepacked_script_key": "source-directory-tarballs/training/meta-textgeneration/prepack/inference-meta-textgeneration/v1.2.0/sourcedir.tar.gz", + "training_prepacked_script_version": "1.2.0", + "training_script_key": "source-directory-tarballs/training/meta-textgeneration/v1.2.0/sourcedir.tar.gz", + "training_supported": true, + "training_volume_size": 256, + "training_vulnerabilities": [], + "training_vulnerable": false, + "url": "https://ai.meta.com/resources/models-and-libraries/llama-downloads/", + "validation_supported": true, + "version": "2.16.2" +} \ No newline at end of file diff --git a/sagemaker-serve/tests/unit/servers/data/jumpstart_specs/openai-reasoning-gpt-oss-20b.json b/sagemaker-serve/tests/unit/servers/data/jumpstart_specs/openai-reasoning-gpt-oss-20b.json new file mode 100644 index 0000000000..b113d8bdff --- /dev/null +++ b/sagemaker-serve/tests/unit/servers/data/jumpstart_specs/openai-reasoning-gpt-oss-20b.json @@ -0,0 +1,3801 @@ +{ + "context_window": "32K-128K", + "default_inference_instance_type": "ml.g7e.2xlarge", + "deprecated": false, + "dynamic_container_deployment_supported": true, + "fine_tuning_supported": false, + "hosting_artifact_compression_type": "None", + "hosting_artifact_key": "openai-reasoning/openai-reasoning-gpt-oss-20b/artifacts/inference/v1.0.0/", + "hosting_artifact_s3_data_type": "S3Prefix", + "hosting_ecr_specs": { + "framework": "vllm", + "framework_version": "0.18.0", + "py_version": "py312" + }, + "hosting_prepacked_artifact_key": "openai-reasoning/openai-reasoning-gpt-oss-20b/artifacts/inference-prepack/v1.0.0/", + "hosting_prepacked_artifact_version": "1.0.0", + "hosting_script_key": "source-directory-tarballs/inference/openai-reasoning/v1.0.0/sourcedir.tar.gz", + "hosting_use_script_uri": false, + "huggingface_id": "openai/gpt-oss-20b", + "incremental_training_supported": false, + "inference_config_components": { + "lmi": { + "default_inference_instance_type": "ml.g7e.2xlarge", + "default_payloads": { + "marketing": { + "body": { + "max_tokens": 200, + "messages": [ + { + "content": "Olly has just promoted you to the marketing department.", + "role": "system" + }, + { + "content": "Create ad text promoting Olly supplements for woman over 40 years using the following keywords: mental, health, sleep, energy, mood, cognitive, adult, children, supplement, and support. Include a slogan.", + "role": "user" + } + ], + "temperature": 0.7, + "top_p": 1 + }, + "content_type": "application/json" + } + }, + "deploy_kwargs": { + "container_startup_health_check_timeout": 1200, + "model_data_download_timeout": 1200 + }, + "hosting_artifact_compression_type": "None", + "hosting_artifact_key": "openai-reasoning/openai-reasoning-gpt-oss-20b/artifacts/inference/v1.0.0/", + "hosting_artifact_s3_data_type": "S3Prefix", + "hosting_ecr_specs": { + "framework": "djl-lmi", + "framework_version": "0.34.0", + "py_version": "py310" + }, + "hosting_instance_type_variants": { + "regional_aliases": { + "af-south-1": { + "alias_ecr_uri_1": "626614931356.dkr.ecr.af-south-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-east-1": { + "alias_ecr_uri_1": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-east-2": { + "alias_ecr_uri_1": "975050140332.dkr.ecr.ap-east-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-northeast-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-northeast-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-northeast-3": { + "alias_ecr_uri_1": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-south-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-south-2": { + "alias_ecr_uri_1": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-southeast-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-southeast-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-southeast-3": { + "alias_ecr_uri_1": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-southeast-4": { + "alias_ecr_uri_1": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-southeast-5": { + "alias_ecr_uri_1": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-southeast-6": { + "alias_ecr_uri_1": "633930458069.dkr.ecr.ap-southeast-6.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ap-southeast-7": { + "alias_ecr_uri_1": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ca-central-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "ca-west-1": { + "alias_ecr_uri_1": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "cn-north-1": { + "alias_ecr_uri_1": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "cn-northwest-1": { + "alias_ecr_uri_1": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "eu-central-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "eu-central-2": { + "alias_ecr_uri_1": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "eu-isoe-west-1": { + "alias_ecr_uri_1": "371248457586.dkr.ecr.eu-isoe-west-1.cloud.adc-e.uk/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "eu-north-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "eu-south-1": { + "alias_ecr_uri_1": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "eu-south-2": { + "alias_ecr_uri_1": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "eu-west-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "eu-west-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "eu-west-3": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "eusc-de-east-1": { + "alias_ecr_uri_1": "204133271717.dkr.ecr.eusc-de-east-1.amazonaws.eu/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "il-central-1": { + "alias_ecr_uri_1": "780543022126.dkr.ecr.il-central-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "me-central-1": { + "alias_ecr_uri_1": "914824155844.dkr.ecr.me-central-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "me-south-1": { + "alias_ecr_uri_1": "217643126080.dkr.ecr.me-south-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "mx-central-1": { + "alias_ecr_uri_1": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "sa-east-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "us-east-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "us-east-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "us-gov-east-1": { + "alias_ecr_uri_1": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "us-gov-west-1": { + "alias_ecr_uri_1": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "us-iso-east-1": { + "alias_ecr_uri_1": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "us-isob-east-1": { + "alias_ecr_uri_1": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "us-isof-east-1": { + "alias_ecr_uri_1": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "us-isof-south-1": { + "alias_ecr_uri_1": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "us-west-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-west-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + }, + "us-west-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128" + } + }, + "variants": { + "g4dn": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g5": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g6": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g6e": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g6f": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g7": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g7e": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "gr6": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "local_gpu": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "ml.g5.12xlarge": { + "properties": { + "environment_variables": { + "MAX_BATCH_SIZE": "16", + "MAX_CONCURRENT_REQUESTS": "32", + "OPTION_DTYPE": "bf16", + "OPTION_ENABLE_CHUNCKED_PREFILL": "True", + "OPTION_GPU_MEMORY_UTILIZATION": "0.88", + "OPTION_MAX_MODEL_LEN": "131072", + "OPTION_MAX_NUM_BATCHED_TOKENS": "131072", + "TENSOR_PARALLEL_DEGREE": "4" + }, + "resource_requirements": { + "min_memory_mb": 98304, + "num_accelerators": 4 + } + } + }, + "ml.g5.24xlarge": { + "properties": { + "environment_variables": { + "MAX_BATCH_SIZE": "16", + "MAX_CONCURRENT_REQUESTS": "32", + "OPTION_DTYPE": "bf16", + "OPTION_ENABLE_CHUNCKED_PREFILL": "True", + "OPTION_GPU_MEMORY_UTILIZATION": "0.88", + "OPTION_MAX_MODEL_LEN": "131072", + "OPTION_MAX_NUM_BATCHED_TOKENS": "131072", + "TENSOR_PARALLEL_DEGREE": "4" + }, + "resource_requirements": { + "min_memory_mb": 196608, + "num_accelerators": 4 + } + } + }, + "ml.g6.12xlarge": { + "properties": { + "environment_variables": { + "MAX_BATCH_SIZE": "16", + "MAX_CONCURRENT_REQUESTS": "32", + "OPTION_DTYPE": "bf16", + "OPTION_ENABLE_CHUNCKED_PREFILL": "True", + "OPTION_GPU_MEMORY_UTILIZATION": "0.88", + "OPTION_MAX_MODEL_LEN": "131072", + "OPTION_MAX_NUM_BATCHED_TOKENS": "131072", + "TENSOR_PARALLEL_DEGREE": "4" + }, + "resource_requirements": { + "min_memory_mb": 98304, + "num_accelerators": 4 + } + } + }, + "ml.g6.24xlarge": { + "properties": { + "environment_variables": { + "MAX_BATCH_SIZE": "16", + "MAX_CONCURRENT_REQUESTS": "32", + "OPTION_DTYPE": "bf16", + "OPTION_ENABLE_CHUNCKED_PREFILL": "True", + "OPTION_GPU_MEMORY_UTILIZATION": "0.88", + "OPTION_MAX_MODEL_LEN": "131072", + "OPTION_MAX_NUM_BATCHED_TOKENS": "131072", + "TENSOR_PARALLEL_DEGREE": "4" + }, + "resource_requirements": { + "min_memory_mb": 196608, + "num_accelerators": 4 + } + } + }, + "ml.g7e.2xlarge": { + "properties": { + "environment_variables": { + "TENSOR_PARALLEL_DEGREE": "1" + }, + "resource_requirements": { + "min_memory_mb": 32768, + "num_accelerators": 1 + } + } + }, + "ml.p5.48xlarge": { + "properties": { + "environment_variables": { + "TENSOR_PARALLEL_DEGREE": "8" + }, + "resource_requirements": { + "min_memory_mb": 1048576, + "num_accelerators": 8 + } + } + }, + "ml.p5e.48xlarge": { + "properties": { + "environment_variables": { + "TENSOR_PARALLEL_DEGREE": "8" + }, + "resource_requirements": { + "min_memory_mb": 1048576, + "num_accelerators": 8 + } + } + }, + "ml.p5en.48xlarge": { + "properties": { + "environment_variables": { + "TENSOR_PARALLEL_DEGREE": "8" + }, + "resource_requirements": { + "min_memory_mb": 1048576, + "num_accelerators": 8 + } + } + }, + "p3": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p3dn": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p4d": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p4de": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p5": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p5e": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p5en": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p6-b200": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "p6-b300": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + } + } + }, + "hosting_prepacked_artifact_key": "openai-reasoning/openai-reasoning-gpt-oss-20b/artifacts/inference-prepack/v1.0.0/", + "hosting_prepacked_artifact_version": "1.0.0", + "hosting_resource_requirements": { + "min_memory_mb": 32768, + "num_accelerators": 1 + }, + "hosting_script_key": "source-directory-tarballs/inference/openai-reasoning/v1.0.0/sourcedir.tar.gz", + "hosting_use_script_uri": false, + "inference_dependencies": [], + "inference_enable_network_isolation": true, + "inference_environment_variables": [ + { + "default": "/opt/ml/model", + "name": "HF_MODEL_ID", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model", + "name": "MODEL_CACHE_ROOT", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "20", + "name": "SAGEMAKER_CONTAINER_LOG_LEVEL", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "1", + "name": "SAGEMAKER_ENV", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "3600", + "name": "SAGEMAKER_MODEL_SERVER_TIMEOUT", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "1", + "name": "SAGEMAKER_MODEL_SERVER_WORKERS", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "inference.py", + "name": "SAGEMAKER_PROGRAM", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model/code", + "name": "SAGEMAKER_SUBMIT_DIRECTORY", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model", + "name": "TIKTOKEN_ENCODINGS_BASE", + "required_for_model_class": true, + "scope": "container", + "type": "text" + } + ], + "inference_vulnerabilities": [], + "inference_vulnerable": false, + "model_kwargs": {}, + "predictor_specs": { + "default_accept_type": "application/json", + "default_content_type": "application/json", + "supported_accept_types": [ + "application/json" + ], + "supported_content_types": [ + "application/json" + ] + }, + "supported_inference_instance_types": [ + "ml.g7e.2xlarge", + "ml.p5.48xlarge", + "ml.p5en.48xlarge", + "ml.p5e.48xlarge", + "ml.g6.12xlarge", + "ml.g5.12xlarge", + "ml.g6.24xlarge", + "ml.g5.24xlarge" + ] + }, + "lmi-optimized": { + "default_inference_instance_type": "ml.g7e.2xlarge", + "default_payloads": { + "marketing": { + "body": { + "max_tokens": 200, + "messages": [ + { + "content": "Olly has just promoted you to the marketing department.", + "role": "system" + }, + { + "content": "Create ad text promoting Olly supplements for woman over 40 years using the following keywords: mental, health, sleep, energy, mood, cognitive, adult, children, supplement, and support. Include a slogan.", + "role": "user" + } + ], + "temperature": 0.7, + "top_p": 1 + }, + "content_type": "application/json" + } + }, + "deploy_kwargs": { + "container_startup_health_check_timeout": 3600, + "model_data_download_timeout": 3600 + }, + "hosting_additional_data_sources": { + "speculative_decoding": [ + { + "artifact_version": "v1.0.0", + "channel_name": "eagle", + "provider": { + "classification": "non-gated", + "name": "JumpStart" + }, + "s3_data_source": { + "compression_type": "None", + "s3_data_type": "S3Prefix", + "s3_uri": "lmi-eagle-heads/models/gpt-oss-20b-p-eagle/" + } + } + ] + }, + "hosting_artifact_compression_type": "None", + "hosting_artifact_key": "openai-reasoning/openai-reasoning-gpt-oss-20b/artifacts/inference/v1.0.0/", + "hosting_artifact_s3_data_type": "S3Prefix", + "hosting_ecr_specs": { + "framework": "vllm", + "framework_version": "0.18.0", + "py_version": "py312" + }, + "hosting_instance_type_variants": { + "regional_aliases": { + "af-south-1": { + "alias_ecr_uri_1": "626614931356.dkr.ecr.af-south-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "626614931356.dkr.ecr.af-south-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-east-1": { + "alias_ecr_uri_1": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-east-2": { + "alias_ecr_uri_1": "975050140332.dkr.ecr.ap-east-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "975050140332.dkr.ecr.ap-east-2.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-northeast-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-northeast-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-northeast-3": { + "alias_ecr_uri_1": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-south-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-south-2": { + "alias_ecr_uri_1": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-southeast-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-southeast-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-southeast-3": { + "alias_ecr_uri_1": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-southeast-4": { + "alias_ecr_uri_1": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-southeast-5": { + "alias_ecr_uri_1": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-southeast-6": { + "alias_ecr_uri_1": "633930458069.dkr.ecr.ap-southeast-6.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "633930458069.dkr.ecr.ap-southeast-6.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ap-southeast-7": { + "alias_ecr_uri_1": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ca-central-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "ca-west-1": { + "alias_ecr_uri_1": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "cn-north-1": { + "alias_ecr_uri_1": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "cn-northwest-1": { + "alias_ecr_uri_1": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "eu-central-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "eu-central-2": { + "alias_ecr_uri_1": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "eu-isoe-west-1": { + "alias_ecr_uri_1": "371248457586.dkr.ecr.eu-isoe-west-1.cloud.adc-e.uk/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "371248457586.dkr.ecr.eu-isoe-west-1.cloud.adc-e.uk/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "eu-north-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "eu-south-1": { + "alias_ecr_uri_1": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "eu-south-2": { + "alias_ecr_uri_1": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "eu-west-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "eu-west-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "eu-west-3": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "eusc-de-east-1": { + "alias_ecr_uri_1": "204133271717.dkr.ecr.eusc-de-east-1.amazonaws.eu/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "204133271717.dkr.ecr.eusc-de-east-1.amazonaws.eu/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "il-central-1": { + "alias_ecr_uri_1": "780543022126.dkr.ecr.il-central-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "780543022126.dkr.ecr.il-central-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "me-central-1": { + "alias_ecr_uri_1": "914824155844.dkr.ecr.me-central-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "914824155844.dkr.ecr.me-central-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "me-south-1": { + "alias_ecr_uri_1": "217643126080.dkr.ecr.me-south-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "217643126080.dkr.ecr.me-south-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "mx-central-1": { + "alias_ecr_uri_1": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "sa-east-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "us-east-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-east-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "us-east-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-east-2.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "us-gov-east-1": { + "alias_ecr_uri_1": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "us-gov-west-1": { + "alias_ecr_uri_1": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "us-iso-east-1": { + "alias_ecr_uri_1": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "us-isob-east-1": { + "alias_ecr_uri_1": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "us-isof-east-1": { + "alias_ecr_uri_1": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "us-isof-south-1": { + "alias_ecr_uri_1": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "us-west-1": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-west-1.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-west-1.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + }, + "us-west-2": { + "alias_ecr_uri_1": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.34.0-lmi16.0.0-cu128", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-west-2.amazonaws.com/vllm:0.18.0-gpu-py312-cu129-ubuntu22.04-sagemaker" + } + }, + "variants": { + "g4dn": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g5": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_1" + } + }, + "g6": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g6e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g6f": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g7": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g7e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "gr6": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "local_gpu": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "ml.g5.12xlarge": { + "properties": { + "environment_variables": { + "MAX_BATCH_SIZE": "16", + "MAX_CONCURRENT_REQUESTS": "32", + "OPTION_DTYPE": "bf16", + "OPTION_ENABLE_CHUNCKED_PREFILL": "True", + "OPTION_GPU_MEMORY_UTILIZATION": "0.88", + "OPTION_MAX_MODEL_LEN": "131072", + "OPTION_MAX_NUM_BATCHED_TOKENS": "131072", + "TENSOR_PARALLEL_DEGREE": "4" + }, + "resource_requirements": { + "min_memory_mb": 98304, + "num_accelerators": 4 + } + } + }, + "ml.g5.24xlarge": { + "properties": { + "environment_variables": { + "MAX_BATCH_SIZE": "16", + "MAX_CONCURRENT_REQUESTS": "32", + "OPTION_DTYPE": "bf16", + "OPTION_ENABLE_CHUNCKED_PREFILL": "True", + "OPTION_GPU_MEMORY_UTILIZATION": "0.88", + "OPTION_MAX_MODEL_LEN": "131072", + "OPTION_MAX_NUM_BATCHED_TOKENS": "131072", + "TENSOR_PARALLEL_DEGREE": "4" + }, + "resource_requirements": { + "min_memory_mb": 196608, + "num_accelerators": 4 + } + } + }, + "ml.g6.12xlarge": { + "properties": { + "environment_variables": { + "MAX_BATCH_SIZE": "16", + "MAX_CONCURRENT_REQUESTS": "32", + "OPTION_DTYPE": "bf16", + "OPTION_ENABLE_CHUNCKED_PREFILL": "True", + "OPTION_GPU_MEMORY_UTILIZATION": "0.88", + "OPTION_MAX_MODEL_LEN": "131072", + "OPTION_MAX_NUM_BATCHED_TOKENS": "131072", + "TENSOR_PARALLEL_DEGREE": "4" + }, + "resource_requirements": { + "min_memory_mb": 98304, + "num_accelerators": 4 + } + } + }, + "ml.g6.24xlarge": { + "properties": { + "environment_variables": { + "MAX_BATCH_SIZE": "16", + "MAX_CONCURRENT_REQUESTS": "32", + "OPTION_DTYPE": "bf16", + "OPTION_ENABLE_CHUNCKED_PREFILL": "True", + "OPTION_GPU_MEMORY_UTILIZATION": "0.88", + "OPTION_MAX_MODEL_LEN": "131072", + "OPTION_MAX_NUM_BATCHED_TOKENS": "131072", + "TENSOR_PARALLEL_DEGREE": "4" + }, + "resource_requirements": { + "min_memory_mb": 196608, + "num_accelerators": 4 + } + } + }, + "ml.g7e.2xlarge": { + "properties": { + "environment_variables": { + "TENSOR_PARALLEL_DEGREE": "1" + }, + "resource_requirements": { + "min_memory_mb": 32768, + "num_accelerators": 1 + } + } + }, + "ml.p5.48xlarge": { + "properties": { + "environment_variables": { + "TENSOR_PARALLEL_DEGREE": "8" + }, + "resource_requirements": { + "min_memory_mb": 1048576, + "num_accelerators": 8 + } + } + }, + "ml.p5e.48xlarge": { + "properties": { + "environment_variables": { + "TENSOR_PARALLEL_DEGREE": "8" + }, + "resource_requirements": { + "min_memory_mb": 1048576, + "num_accelerators": 8 + } + } + }, + "ml.p5en.48xlarge": { + "properties": { + "environment_variables": { + "TENSOR_PARALLEL_DEGREE": "8" + }, + "resource_requirements": { + "min_memory_mb": 1048576, + "num_accelerators": 8 + } + } + }, + "p3": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p3dn": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p4d": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p4de": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p5": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p5e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p5en": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p6-b200": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p6-b300": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + } + } + }, + "hosting_prepacked_artifact_key": "openai-reasoning/openai-reasoning-gpt-oss-20b/artifacts/inference-prepack/v1.0.0/", + "hosting_prepacked_artifact_version": "1.0.0", + "hosting_resource_requirements": { + "min_memory_mb": 32768, + "num_accelerators": 1 + }, + "hosting_script_key": "source-directory-tarballs/inference/openai-reasoning/v1.0.0/sourcedir.tar.gz", + "hosting_use_script_uri": false, + "inference_dependencies": [], + "inference_enable_network_isolation": true, + "inference_environment_variables": [ + { + "default": "/opt/ml/model", + "name": "HF_MODEL_ID", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model", + "name": "MODEL_CACHE_ROOT", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "20", + "name": "SAGEMAKER_CONTAINER_LOG_LEVEL", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "1", + "name": "SAGEMAKER_ENV", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "3600", + "name": "SAGEMAKER_MODEL_SERVER_TIMEOUT", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": 1, + "name": "SAGEMAKER_MODEL_SERVER_WORKERS", + "required_for_model_class": true, + "scope": "container", + "type": "int" + }, + { + "default": "inference.py", + "name": "SAGEMAKER_PROGRAM", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model/code", + "name": "SAGEMAKER_SUBMIT_DIRECTORY", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "8192", + "name": "SM_VLLM_MAX_MODEL_LEN", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "{\"model\": \"/opt/ml/additional-model-data-sources/eagle\", \"method\": \"eagle3\", \"num_speculative_tokens\": 3, \"parallel_drafting\": true}", + "name": "SM_VLLM_SPECULATIVE_CONFIG", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "1", + "name": "SM_VLLM_TENSOR_PARALLEL_SIZE", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model", + "name": "TIKTOKEN_ENCODINGS_BASE", + "required_for_model_class": true, + "scope": "container", + "type": "text" + } + ], + "inference_vulnerabilities": [], + "inference_vulnerable": false, + "model_kwargs": {}, + "predictor_specs": { + "default_accept_type": "application/json", + "default_content_type": "application/json", + "supported_accept_types": [ + "application/json" + ], + "supported_content_types": [ + "application/json" + ] + }, + "supported_inference_instance_types": [ + "ml.g7e.2xlarge", + "ml.p5.48xlarge", + "ml.p5en.48xlarge", + "ml.p5e.48xlarge", + "ml.g6.12xlarge", + "ml.g5.12xlarge", + "ml.g6.24xlarge", + "ml.g5.24xlarge" + ] + } + }, + "inference_config_rankings": { + "overall": { + "description": "default", + "rankings": [ + "lmi-optimized", + "lmi" + ] + } + }, + "inference_configs": { + "lmi": { + "component_names": [ + "lmi" + ] + }, + "lmi-optimized": { + "acceleration_configs": [ + { + "enabled": true, + "type": "Speculative-Decoding" + } + ], + "benchmark_metrics": { + "ml.g7e.2xlarge": [ + { + "concurrency": "1", + "name": "throughput", + "unit": "tokens/sec", + "value": "40" + }, + { + "concurrency": "1", + "name": "latency", + "unit": "sec", + "value": "50.0" + }, + { + "concurrency": "2", + "name": "throughput", + "unit": "tokens/sec", + "value": "80" + }, + { + "concurrency": "2", + "name": "latency", + "unit": "sec", + "value": "25.0" + }, + { + "concurrency": "4", + "name": "throughput", + "unit": "tokens/sec", + "value": "160" + }, + { + "concurrency": "4", + "name": "latency", + "unit": "sec", + "value": "12.5" + }, + { + "concurrency": "8", + "name": "throughput", + "unit": "tokens/sec", + "value": "320" + }, + { + "concurrency": "8", + "name": "latency", + "unit": "sec", + "value": "6.25" + }, + { + "concurrency": "16", + "name": "throughput", + "unit": "tokens/sec", + "value": "320" + }, + { + "concurrency": "16", + "name": "latency", + "unit": "sec", + "value": "3.12" + }, + { + "concurrency": "32", + "name": "throughput", + "unit": "tokens/sec", + "value": "320" + }, + { + "concurrency": "32", + "name": "latency", + "unit": "sec", + "value": "1.56" + }, + { + "concurrency": "64", + "name": "throughput", + "unit": "tokens/sec", + "value": "320" + }, + { + "concurrency": "64", + "name": "latency", + "unit": "sec", + "value": "1.0" + }, + { + "concurrency": "128", + "name": "throughput", + "unit": "tokens/sec", + "value": "320" + }, + { + "concurrency": "128", + "name": "latency", + "unit": "sec", + "value": "1.0" + } + ] + }, + "component_names": [ + "lmi-optimized" + ] + } + }, + "inference_dependencies": [], + "inference_enable_network_isolation": true, + "inference_environment_variables": [ + { + "default": "/opt/ml/model", + "name": "HF_MODEL_ID", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model", + "name": "MODEL_CACHE_ROOT", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "20", + "name": "SAGEMAKER_CONTAINER_LOG_LEVEL", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "1", + "name": "SAGEMAKER_ENV", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "3600", + "name": "SAGEMAKER_MODEL_SERVER_TIMEOUT", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": 1, + "name": "SAGEMAKER_MODEL_SERVER_WORKERS", + "required_for_model_class": true, + "scope": "container", + "type": "int" + }, + { + "default": "inference.py", + "name": "SAGEMAKER_PROGRAM", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model/code", + "name": "SAGEMAKER_SUBMIT_DIRECTORY", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "8192", + "name": "SM_VLLM_MAX_MODEL_LEN", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "{\"model\": \"/opt/ml/additional-model-data-sources/eagle\", \"method\": \"eagle3\", \"num_speculative_tokens\": 3, \"parallel_drafting\": true}", + "name": "SM_VLLM_SPECULATIVE_CONFIG", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "1", + "name": "SM_VLLM_TENSOR_PARALLEL_SIZE", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model", + "name": "TIKTOKEN_ENCODINGS_BASE", + "required_for_model_class": true, + "scope": "container", + "type": "text" + } + ], + "inference_vulnerabilities": [], + "inference_vulnerable": false, + "input_modalities": [ + "Text" + ], + "languages": [ + "English" + ], + "license": "Apache 2.0", + "metrics": [], + "min_sdk_version": "2.244.2", + "ml_framework": "PyTorch", + "model_access": "Open Weight", + "model_id": "openai-reasoning-gpt-oss-20b", + "model_kwargs": {}, + "model_size": "10B-70B", + "output_modalities": [ + "Text" + ], + "provider": "openai", + "recipe_collection": [ + { + "customization_technique": "DPO", + "display_name": "GPT OSS 20B Direct Preference Optimization on GPU, 4K sequence length", + "hardware": "GPU", + "hosting_configs": [ + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 196608, + "number_of_accelerator_devices_required": 4, + "number_of_cpu_cores_required": 36 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENABLE_LORA": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_CPU_LORAS": "32", + "OPTION_MAX_LORAS": "4", + "OPTION_MAX_LORA_RANK": "128", + "OPTION_MAX_ROLLING_BATCH_SIZE": "4", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "4", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1", + "SAGEMAKER_MAX_NUMBER_OF_ADAPTERS_IN_MEMORY": "64" + }, + "instance_type": "ml.g6e.12xlarge", + "profile": "Default" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1024000, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENABLE_LORA": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_CPU_LORAS": "64", + "OPTION_MAX_LORAS": "8", + "OPTION_MAX_LORA_RANK": "128", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1", + "SAGEMAKER_MAX_NUMBER_OF_ADAPTERS_IN_MEMORY": "128" + }, + "instance_type": "ml.g6e.48xlarge" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1539316, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENABLE_LORA": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_CPU_LORAS": "64", + "OPTION_MAX_LORAS": "8", + "OPTION_MAX_LORA_RANK": "128", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1", + "SAGEMAKER_MAX_NUMBER_OF_ADAPTERS_IN_MEMORY": "128" + }, + "instance_type": "ml.p5.48xlarge" + } + ], + "hp_eks_override_params_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_dpo_override_params_k8s_v2.2.4.json", + "hp_eks_payload_template_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_dpo_payload_template_k8s_v2.2.4.yaml", + "instance_count": 1, + "is_subscription_model": false, + "name": "llmft_gpt_oss_20b_seq4k_gpu_dpo", + "peft": "LORA", + "recipe_file_path": "recipes/fine-tuning/gpt_oss/llmft_gpt_oss_20b_seq4k_gpu_dpo.yaml", + "regional_parameters": { + "beta": { + "us-west-2": { + "hp_eks_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-DPO-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + } + }, + "gamma": { + "us-east-1": { + "hp_eks_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-DPO-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-DPO-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + } + }, + "prod": { + "af-south-1": { + "inference": "626614931356.dkr.ecr.af-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-east-1": { + "inference": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-1": { + "hp_eks_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "APN1-ServerlessTraining:FineTuning-DPO-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "ap-northeast-2": { + "inference": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-3": { + "inference": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-1": { + "hp_eks_regional_ecr_uri": "423350936952.dkr.ecr.ap-south-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-2": { + "inference": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-1": { + "hp_eks_regional_ecr_uri": "885852567298.dkr.ecr.ap-southeast-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-2": { + "hp_eks_regional_ecr_uri": "304708117039.dkr.ecr.ap-southeast-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-3": { + "inference": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-4": { + "inference": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-5": { + "inference": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-7": { + "inference": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-central-1": { + "inference": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-west-1": { + "inference": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-north-1": { + "inference": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-northwest-1": { + "inference": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-1": { + "hp_eks_regional_ecr_uri": "391061375763.dkr.ecr.eu-central-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-2": { + "inference": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-north-1": { + "hp_eks_regional_ecr_uri": "963403601044.dkr.ecr.eu-north-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-1": { + "inference": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-2": { + "hp_eks_regional_ecr_uri": "330290781619.dkr.ecr.eu-south-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-1": { + "hp_eks_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "EU-ServerlessTraining:FineTuning-DPO-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "eu-west-2": { + "hp_eks_regional_ecr_uri": "016839105697.dkr.ecr.eu-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-3": { + "inference": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "il-central-1": { + "inference": "780543022126.dkr.ecr.il-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-central-1": { + "inference": "914824155844.dkr.ecr.me-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-south-1": { + "inference": "217643126080.dkr.ecr.me-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "mx-central-1": { + "inference": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "sa-east-1": { + "hp_eks_regional_ecr_uri": "311136344257.dkr.ecr.sa-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-east-1": { + "hp_eks_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-DPO-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "us-east-2": { + "hp_eks_regional_ecr_uri": "556809692997.dkr.ecr.us-east-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-east-1": { + "inference": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-west-1": { + "inference": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-iso-east-1": { + "inference": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isob-east-1": { + "inference": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-east-1": { + "inference": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-south-1": { + "inference": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-1": { + "hp_eks_regional_ecr_uri": "827510180725.dkr.ecr.us-west-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-DPO-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + } + } + }, + "sequence_length": "4K", + "serverless_metering_type": "Token-based", + "smtj_override_params_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_dpo_override_params_sm_jobs_v2.2.4.json", + "smtj_recipe_template_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_dpo_payload_template_sm_jobs_v2.2.4.yaml", + "supported_instance_types": [ + "ml.p5.48xlarge", + "ml.p5e.48xlarge", + "ml.p5en.48xlarge" + ], + "type": "FineTuning", + "versions": [ + "2.2.4" + ] + }, + { + "customization_technique": "SFT", + "display_name": "GPT OSS 20B Supervised Fine-Tuning with LoRA on GPU, 4K sequence length", + "hardware": "GPU", + "hosting_configs": [ + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 196608, + "number_of_accelerator_devices_required": 4, + "number_of_cpu_cores_required": 36 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENABLE_LORA": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_CPU_LORAS": "32", + "OPTION_MAX_LORAS": "4", + "OPTION_MAX_LORA_RANK": "128", + "OPTION_MAX_ROLLING_BATCH_SIZE": "4", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "4", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1", + "SAGEMAKER_MAX_NUMBER_OF_ADAPTERS_IN_MEMORY": "64" + }, + "instance_type": "ml.g6e.12xlarge", + "profile": "Default" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1024000, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENABLE_LORA": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_CPU_LORAS": "64", + "OPTION_MAX_LORAS": "8", + "OPTION_MAX_LORA_RANK": "128", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1", + "SAGEMAKER_MAX_NUMBER_OF_ADAPTERS_IN_MEMORY": "128" + }, + "instance_type": "ml.g6e.48xlarge" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1539316, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENABLE_LORA": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_CPU_LORAS": "64", + "OPTION_MAX_LORAS": "8", + "OPTION_MAX_LORA_RANK": "128", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1", + "SAGEMAKER_MAX_NUMBER_OF_ADAPTERS_IN_MEMORY": "128" + }, + "instance_type": "ml.p5.48xlarge" + } + ], + "hp_eks_override_params_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_sft_lora_override_params_k8s_v2.3.1.json", + "hp_eks_payload_template_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_sft_lora_payload_template_k8s_v2.3.1.yaml", + "instance_count": 1, + "is_subscription_model": false, + "name": "llmft_gpt_oss_20b_seq4k_gpu_sft_lora", + "peft": "LORA", + "recipe_file_path": "recipes/fine-tuning/gpt_oss/llmft_gpt_oss_20b_seq4k_gpu_sft_lora.yaml", + "regional_parameters": { + "beta": { + "us-west-2": { + "hp_eks_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-SFT-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + } + }, + "gamma": { + "us-east-1": { + "hp_eks_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-SFT-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-SFT-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + } + }, + "prod": { + "af-south-1": { + "inference": "626614931356.dkr.ecr.af-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-east-1": { + "inference": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-1": { + "hp_eks_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "APN1-ServerlessTraining:FineTuning-SFT-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "ap-northeast-2": { + "inference": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-3": { + "inference": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-1": { + "hp_eks_regional_ecr_uri": "423350936952.dkr.ecr.ap-south-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-2": { + "inference": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-1": { + "hp_eks_regional_ecr_uri": "885852567298.dkr.ecr.ap-southeast-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-2": { + "hp_eks_regional_ecr_uri": "304708117039.dkr.ecr.ap-southeast-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-3": { + "inference": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-4": { + "inference": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-5": { + "inference": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-7": { + "inference": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-central-1": { + "inference": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-west-1": { + "inference": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-north-1": { + "inference": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-northwest-1": { + "inference": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-1": { + "hp_eks_regional_ecr_uri": "391061375763.dkr.ecr.eu-central-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-2": { + "inference": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-north-1": { + "hp_eks_regional_ecr_uri": "963403601044.dkr.ecr.eu-north-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-1": { + "inference": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-2": { + "hp_eks_regional_ecr_uri": "330290781619.dkr.ecr.eu-south-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-1": { + "hp_eks_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "EU-ServerlessTraining:FineTuning-SFT-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "eu-west-2": { + "hp_eks_regional_ecr_uri": "016839105697.dkr.ecr.eu-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-3": { + "inference": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "il-central-1": { + "inference": "780543022126.dkr.ecr.il-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-central-1": { + "inference": "914824155844.dkr.ecr.me-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-south-1": { + "inference": "217643126080.dkr.ecr.me-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "mx-central-1": { + "inference": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "sa-east-1": { + "hp_eks_regional_ecr_uri": "311136344257.dkr.ecr.sa-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-east-1": { + "hp_eks_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-SFT-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "us-east-2": { + "hp_eks_regional_ecr_uri": "556809692997.dkr.ecr.us-east-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-east-1": { + "inference": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-west-1": { + "inference": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-iso-east-1": { + "inference": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isob-east-1": { + "inference": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-east-1": { + "inference": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-south-1": { + "inference": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-1": { + "hp_eks_regional_ecr_uri": "827510180725.dkr.ecr.us-west-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-SFT-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + } + } + }, + "sequence_length": "4K", + "serverless_metering_type": "Token-based", + "smtj_override_params_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_sft_lora_override_params_sm_jobs_v2.3.1.json", + "smtj_recipe_template_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_sft_lora_payload_template_sm_jobs_v2.3.1.yaml", + "supported_instance_types": [ + "ml.p5.48xlarge", + "ml.p5e.48xlarge", + "ml.p5en.48xlarge" + ], + "type": "FineTuning", + "versions": [ + "2.3.1" + ] + }, + { + "customization_technique": "RLAIF", + "display_name": "GPT OSS 20B GRPO RLAIF Fine-Tuning with LoRA", + "hardware": "GPU", + "hosting_configs": [ + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 196608, + "number_of_accelerator_devices_required": 4, + "number_of_cpu_cores_required": 36 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENABLE_LORA": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_CPU_LORAS": "32", + "OPTION_MAX_LORAS": "4", + "OPTION_MAX_LORA_RANK": "128", + "OPTION_MAX_ROLLING_BATCH_SIZE": "4", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "4", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1", + "SAGEMAKER_MAX_NUMBER_OF_ADAPTERS_IN_MEMORY": "64" + }, + "instance_type": "ml.g6e.12xlarge", + "profile": "Default" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1539316, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENABLE_LORA": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_CPU_LORAS": "64", + "OPTION_MAX_LORAS": "8", + "OPTION_MAX_LORA_RANK": "128", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1", + "SAGEMAKER_MAX_NUMBER_OF_ADAPTERS_IN_MEMORY": "128" + }, + "instance_type": "ml.p5.48xlarge" + } + ], + "hp_eks_override_params_s3_key": "recipes/verl-grpo-rlaif-gpt-oss-20b-lora_override_params_k8s_v2.2.3.json", + "hp_eks_payload_template_s3_key": "recipes/verl-grpo-rlaif-gpt-oss-20b-lora_payload_template_k8s_v2.2.3.yaml", + "instance_count": 1, + "is_subscription_model": false, + "name": "verl-grpo-rlaif-gpt-oss-20b-lora", + "peft": "LORA", + "recipe_file_path": "recipes/fine-tuning/gpt_oss/verl-grpo-rlaif-gpt-oss-20b-lora.yaml", + "regional_parameters": { + "beta": { + "us-west-2": { + "hp_eks_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-RLAIF-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + } + }, + "gamma": { + "us-east-1": { + "hp_eks_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-RLAIF-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-RLAIF-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + } + }, + "prod": { + "af-south-1": { + "inference": "626614931356.dkr.ecr.af-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-east-1": { + "inference": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-1": { + "hp_eks_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "APN1-ServerlessTraining:FineTuning-RLAIF-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "ap-northeast-2": { + "inference": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-3": { + "inference": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-1": { + "hp_eks_regional_ecr_uri": "423350936952.dkr.ecr.ap-south-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-2": { + "inference": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-1": { + "hp_eks_regional_ecr_uri": "885852567298.dkr.ecr.ap-southeast-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-2": { + "hp_eks_regional_ecr_uri": "304708117039.dkr.ecr.ap-southeast-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-3": { + "inference": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-4": { + "inference": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-5": { + "inference": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-7": { + "inference": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-central-1": { + "inference": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-west-1": { + "inference": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-north-1": { + "inference": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-northwest-1": { + "inference": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-1": { + "hp_eks_regional_ecr_uri": "391061375763.dkr.ecr.eu-central-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-2": { + "inference": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-north-1": { + "hp_eks_regional_ecr_uri": "963403601044.dkr.ecr.eu-north-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-1": { + "inference": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-2": { + "hp_eks_regional_ecr_uri": "330290781619.dkr.ecr.eu-south-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-1": { + "hp_eks_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "EU-ServerlessTraining:FineTuning-RLAIF-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "eu-west-2": { + "hp_eks_regional_ecr_uri": "016839105697.dkr.ecr.eu-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-3": { + "inference": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "il-central-1": { + "inference": "780543022126.dkr.ecr.il-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-central-1": { + "inference": "914824155844.dkr.ecr.me-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-south-1": { + "inference": "217643126080.dkr.ecr.me-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "mx-central-1": { + "inference": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "sa-east-1": { + "hp_eks_regional_ecr_uri": "311136344257.dkr.ecr.sa-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-east-1": { + "hp_eks_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-RLAIF-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "us-east-2": { + "hp_eks_regional_ecr_uri": "556809692997.dkr.ecr.us-east-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-east-1": { + "inference": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-west-1": { + "inference": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-iso-east-1": { + "inference": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isob-east-1": { + "inference": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-east-1": { + "inference": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-south-1": { + "inference": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-1": { + "hp_eks_regional_ecr_uri": "827510180725.dkr.ecr.us-west-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-RLAIF-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + } + } + }, + "sequence_length": "4K", + "serverless_metering_type": "Hourly", + "smtj_override_params_s3_key": "recipes/verl-grpo-rlaif-gpt-oss-20b-lora_override_params_sm_jobs_v2.2.3.json", + "smtj_recipe_template_s3_key": "recipes/verl-grpo-rlaif-gpt-oss-20b-lora_payload_template_sm_jobs_v2.2.3.yaml", + "supported_instance_types": [ + "ml.p5.48xlarge" + ], + "type": "FineTuning", + "versions": [ + "2.2.3" + ] + }, + { + "customization_technique": "RLVR", + "display_name": "GPT OSS 20B GRPO RLVR Fine-Tuning with LoRA", + "hardware": "GPU", + "hosting_configs": [ + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 196608, + "number_of_accelerator_devices_required": 4, + "number_of_cpu_cores_required": 36 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENABLE_LORA": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_CPU_LORAS": "32", + "OPTION_MAX_LORAS": "4", + "OPTION_MAX_LORA_RANK": "128", + "OPTION_MAX_ROLLING_BATCH_SIZE": "4", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "4", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1", + "SAGEMAKER_MAX_NUMBER_OF_ADAPTERS_IN_MEMORY": "64" + }, + "instance_type": "ml.g6e.12xlarge", + "profile": "Default" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1539316, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENABLE_LORA": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_CPU_LORAS": "64", + "OPTION_MAX_LORAS": "8", + "OPTION_MAX_LORA_RANK": "128", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1", + "SAGEMAKER_MAX_NUMBER_OF_ADAPTERS_IN_MEMORY": "128" + }, + "instance_type": "ml.p5.48xlarge" + } + ], + "hp_eks_override_params_s3_key": "recipes/verl-grpo-rlvr-gpt-oss-20b-lora_override_params_k8s_v2.2.2.json", + "hp_eks_payload_template_s3_key": "recipes/verl-grpo-rlvr-gpt-oss-20b-lora_payload_template_k8s_v2.2.2.yaml", + "instance_count": 1, + "is_subscription_model": false, + "name": "verl-grpo-rlvr-gpt-oss-20b-lora", + "peft": "LORA", + "recipe_file_path": "recipes/fine-tuning/gpt_oss/verl-grpo-rlvr-gpt-oss-20b-lora.yaml", + "regional_parameters": { + "beta": { + "us-west-2": { + "hp_eks_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-RLVR-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + } + }, + "gamma": { + "us-east-1": { + "hp_eks_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-RLVR-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-RLVR-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + } + }, + "prod": { + "af-south-1": { + "inference": "626614931356.dkr.ecr.af-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-east-1": { + "inference": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-1": { + "hp_eks_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "APN1-ServerlessTraining:FineTuning-RLVR-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "ap-northeast-2": { + "inference": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-3": { + "inference": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-1": { + "hp_eks_regional_ecr_uri": "423350936952.dkr.ecr.ap-south-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-2": { + "inference": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-1": { + "hp_eks_regional_ecr_uri": "885852567298.dkr.ecr.ap-southeast-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-2": { + "hp_eks_regional_ecr_uri": "304708117039.dkr.ecr.ap-southeast-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-3": { + "inference": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-4": { + "inference": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-5": { + "inference": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-7": { + "inference": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-central-1": { + "inference": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-west-1": { + "inference": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-north-1": { + "inference": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-northwest-1": { + "inference": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-1": { + "hp_eks_regional_ecr_uri": "391061375763.dkr.ecr.eu-central-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-2": { + "inference": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-north-1": { + "hp_eks_regional_ecr_uri": "963403601044.dkr.ecr.eu-north-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-1": { + "inference": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-2": { + "hp_eks_regional_ecr_uri": "330290781619.dkr.ecr.eu-south-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-1": { + "hp_eks_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "EU-ServerlessTraining:FineTuning-RLVR-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "eu-west-2": { + "hp_eks_regional_ecr_uri": "016839105697.dkr.ecr.eu-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-3": { + "inference": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "il-central-1": { + "inference": "780543022126.dkr.ecr.il-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-central-1": { + "inference": "914824155844.dkr.ecr.me-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-south-1": { + "inference": "217643126080.dkr.ecr.me-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "mx-central-1": { + "inference": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "sa-east-1": { + "hp_eks_regional_ecr_uri": "311136344257.dkr.ecr.sa-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-east-1": { + "hp_eks_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-RLVR-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "us-east-2": { + "hp_eks_regional_ecr_uri": "556809692997.dkr.ecr.us-east-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-east-1": { + "inference": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-west-1": { + "inference": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-iso-east-1": { + "inference": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isob-east-1": { + "inference": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-east-1": { + "inference": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-south-1": { + "inference": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-1": { + "hp_eks_regional_ecr_uri": "827510180725.dkr.ecr.us-west-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-RLVR-LORA-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + } + } + }, + "sequence_length": "1K", + "serverless_metering_type": "Hourly", + "smtj_override_params_s3_key": "recipes/verl-grpo-rlvr-gpt-oss-20b-lora_override_params_sm_jobs_v2.2.2.json", + "smtj_recipe_template_s3_key": "recipes/verl-grpo-rlvr-gpt-oss-20b-lora_payload_template_sm_jobs_v2.2.2.yaml", + "supported_instance_types": [ + "ml.p5.48xlarge" + ], + "type": "FineTuning", + "versions": [ + "2.2.2" + ] + }, + { + "display_name": "Open Source Evaluation Evaluation on GPU - Openai Reasoning Gpt Oss 20B", + "evaluation_type": "DeterministicEvaluation", + "hardware": "GPU", + "hp_eks_override_params_s3_key": "recipes/openai_reasoning_gpt_oss_20b_override_params_k8s_v2.0.2.json", + "hp_eks_payload_template_s3_key": "recipes/openai_reasoning_gpt_oss_20b_payload_template_k8s_v2.0.2.yaml", + "name": "open-source-eval-openai-reasoning-gpt-oss-20b-deterministic", + "regional_parameters": { + "gamma": { + "ap-northeast-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "APN1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "APN1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "ap-southeast-2": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "APS2-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "APS2-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:latest" + }, + "eu-central-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "EUC1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "EUC1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "eu-west-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "EUW1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "EUW1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "us-east-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "USE1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "USE1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "USW2-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "USW2-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:latest" + } + }, + "prod": { + "ap-northeast-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "APN1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "APN1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "ap-southeast-2": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "APS2-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "APS2-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:latest" + }, + "eu-central-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "EUC1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "EUC1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "eu-west-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "EUW1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "EUW1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "us-east-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "USE1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "USE1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "USW2-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "USW2-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:latest" + } + } + }, + "smtj_override_params_s3_key": "recipes/open-source-eval-openai-reasoning-gpt-oss-20b-deterministic_override_params_sm_jobs_v2.0.2.json", + "smtj_recipe_template_s3_key": "recipes/open-source-eval-openai-reasoning-gpt-oss-20b-deterministic_payload_template_sm_jobs_v2.0.2.yaml", + "supported_instance_types": [ + "ml.g5.12xlarge", + "ml.g5.24xlarge", + "ml.g5.48xlarge", + "ml.g6.24xlarge", + "ml.g6.48xlarge", + "ml.p4d.24xlarge", + "ml.p4de.24xlarge" + ], + "type": "Evaluation", + "versions": [ + "2.0.1" + ] + }, + { + "display_name": "Open Source Evaluation Evaluation on GPU - Openai Reasoning Gpt Oss 20B", + "evaluation_type": "LLMAJEvaluation", + "hardware": "CPU", + "hp_eks_override_params_s3_key": "recipes/openai_reasoning_gpt_oss_20b_override_params_k8s_v2.0.2.json", + "hp_eks_payload_template_s3_key": "recipes/openai_reasoning_gpt_oss_20b_payload_template_k8s_v2.0.2.yaml", + "name": "open-source-eval-openai-reasoning-gpt-oss-20b-llmaj", + "regional_parameters": { + "gamma": { + "ap-northeast-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "APN1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "APN1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "ap-southeast-2": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "APS2-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "APS2-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:latest" + }, + "eu-central-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "EUC1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "EUC1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "eu-west-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "EUW1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "EUW1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "us-east-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "USE1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "USE1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "USW2-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "USW2-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:latest" + } + }, + "prod": { + "ap-northeast-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "APN1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "APN1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "ap-southeast-2": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "APS2-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "APS2-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/sagemaker-evaluation:latest" + }, + "eu-central-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "EUC1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "EUC1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "eu-west-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "EUW1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "EUW1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "us-east-1": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "USE1-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "USE1-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.us-east-1.amazonaws.com/sagemaker-evaluation:latest" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:latest", + "serverless_sku_input": "USW2-ServerlessTraining:Evaluation-Input-openai-reasoning-gpt-oss-20b", + "serverless_sku_output": "USW2-ServerlessTraining:Evaluation-Output-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "763104351884.dkr.ecr.us-west-2.amazonaws.com/sagemaker-evaluation:latest" + } + } + }, + "smtj_override_params_s3_key": "recipes/open-source-eval-openai-reasoning-gpt-oss-20b-llmaj_override_params_sm_jobs_v2.0.2.json", + "smtj_recipe_template_s3_key": "recipes/open-source-eval-openai-reasoning-gpt-oss-20b-llmaj_payload_template_sm_jobs_v2.0.2.yaml", + "supported_instance_types": [ + "ml.t3.large" + ], + "type": "Evaluation", + "versions": [ + "2.0.1" + ] + }, + { + "customization_technique": "MTRL", + "display_name": "Agentic RFT for GPT OSS 20B Lora", + "hosting_configs": [ + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 196608, + "number_of_accelerator_devices_required": 4, + "number_of_cpu_cores_required": 36 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENABLE_LORA": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_CPU_LORAS": "32", + "OPTION_MAX_LORAS": "4", + "OPTION_MAX_LORA_RANK": "256", + "OPTION_MAX_ROLLING_BATCH_SIZE": "4", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "4", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1", + "SAGEMAKER_MAX_NUMBER_OF_ADAPTERS_IN_MEMORY": "64" + }, + "instance_type": "ml.g6e.12xlarge", + "profile": "Default" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1024000, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENABLE_LORA": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_CPU_LORAS": "64", + "OPTION_MAX_LORAS": "8", + "OPTION_MAX_LORA_RANK": "256", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1", + "SAGEMAKER_MAX_NUMBER_OF_ADAPTERS_IN_MEMORY": "128" + }, + "instance_type": "ml.g6e.48xlarge" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1539316, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENABLE_LORA": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_CPU_LORAS": "64", + "OPTION_MAX_LORAS": "8", + "OPTION_MAX_LORA_RANK": "256", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1", + "SAGEMAKER_MAX_NUMBER_OF_ADAPTERS_IN_MEMORY": "128" + }, + "instance_type": "ml.p5.48xlarge" + } + ], + "name": "mtrl-gpt-oss-20b-lora", + "peft": "LORA", + "recipe_file_path": "recipes/fine-tuning/gpt_oss/mtrl-gpt-oss-20b-lora.yaml", + "regional_parameters": { + "beta": { + "us-west-2": { + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130", + "smtj_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/mtrl-recipes:rft-trainer-v1.0.0" + } + }, + "gamma": { + "us-east-1": { + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130", + "smtj_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/mtrl-recipes:rft-trainer-v1.0.0" + }, + "us-west-2": { + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130", + "smtj_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/mtrl-recipes:rft-trainer-v1.0.0" + } + }, + "prod": { + "af-south-1": { + "inference": "626614931356.dkr.ecr.af-south-1.amazonaws.com/djl-inference:0.36.0-lmi24.0.0-cu129" + }, + "ap-east-1": { + "inference": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/djl-inference:0.36.0-lmi24.0.0-cu129" + }, + "ap-northeast-1": { + "inference": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/djl-inference:0.36.0-lmi24.0.0-cu129" + }, + "ap-northeast-2": { + "inference": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/djl-inference:0.36.0-lmi24.0.0-cu129" + }, + "ap-northeast-3": { + "inference": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/djl-inference:0.36.0-lmi24.0.0-cu129" + }, + "ap-south-1": { + "inference": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/djl-inference:0.36.0-lmi24.0.0-cu129" + }, + "ap-south-2": { + "inference": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "ap-southeast-1": { + "inference": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "ap-southeast-2": { + "inference": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "ap-southeast-3": { + "inference": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "ap-southeast-4": { + "inference": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "ap-southeast-5": { + "inference": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "ap-southeast-7": { + "inference": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "ca-central-1": { + "inference": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "ca-west-1": { + "inference": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "cn-north-1": { + "inference": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "cn-northwest-1": { + "inference": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "eu-central-1": { + "inference": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "eu-central-2": { + "inference": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "eu-north-1": { + "inference": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "eu-south-1": { + "inference": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "eu-south-2": { + "inference": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "eu-west-1": { + "inference": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "eu-west-2": { + "inference": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "eu-west-3": { + "inference": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "il-central-1": { + "inference": "780543022126.dkr.ecr.il-central-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "me-central-1": { + "inference": "914824155844.dkr.ecr.me-central-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "me-south-1": { + "inference": "217643126080.dkr.ecr.me-south-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "mx-central-1": { + "inference": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "sa-east-1": { + "inference": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "us-east-1": { + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130", + "smtj_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/mtrl-recipes:rft-trainer-v1.0.0" + }, + "us-east-2": { + "inference": "763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "us-gov-east-1": { + "inference": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "us-gov-west-1": { + "inference": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "us-iso-east-1": { + "inference": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "us-isob-east-1": { + "inference": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "us-isof-east-1": { + "inference": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "us-isof-south-1": { + "inference": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "us-west-1": { + "inference": "763104351884.dkr.ecr.us-west-1.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130" + }, + "us-west-2": { + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.36.0-lmi25.0.0-cu130", + "smtj_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/mtrl-recipes:rft-trainer-v1.0.0" + } + } + }, + "sequence_length": "64K", + "smtj_override_params_s3_key": "recipes/mtrl-gpt-oss-20b-lora_override_params_sm_jobs_v1.0.22.json", + "smtj_recipe_template_s3_key": "recipes/mtrl-gpt-oss-20b-lora_payload_template_sm_jobs_v1.0.22.yaml", + "type": "FineTuning", + "versions": [ + "1.0.22" + ] + }, + { + "display_name": "MTRL Evaluation", + "evaluation_type": "MTRLEvaluation", + "name": "mtrl-eval-openai-reasoning-gpt-oss-20b", + "recipe_file_path": "recipes/evaluation/mtrl/mtrl_eval.yaml", + "regional_parameters": { + "beta": { + "us-west-2": { + "smtj_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/mtrl-recipes:rft-trainer-v1.0.0" + } + }, + "gamma": { + "us-east-1": { + "smtj_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/mtrl-recipes:rft-trainer-v1.0.0" + }, + "us-west-2": { + "smtj_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/mtrl-recipes:rft-trainer-v1.0.0" + } + }, + "prod": { + "us-east-1": { + "smtj_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/mtrl-recipes:rft-trainer-v1.0.0" + }, + "us-west-2": { + "smtj_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/mtrl-recipes:rft-trainer-v1.0.0" + } + } + }, + "smtj_override_params_s3_key": "recipes/mtrl-eval-openai-reasoning-gpt-oss-20b_override_params_sm_jobs_v1.0.12.json", + "smtj_recipe_template_s3_key": "recipes/mtrl-eval-openai-reasoning-gpt-oss-20b_payload_template_sm_jobs_v1.0.12.yaml", + "type": "Evaluation", + "versions": [ + "1.0.12" + ] + }, + { + "customization_technique": "SFT", + "display_name": "GPT-OSS 20B Supervised Fine-Tuning Full Fine-Tuning on GPU", + "hardware": "GPU", + "hosting_configs": [ + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 196608, + "number_of_accelerator_devices_required": 4, + "number_of_cpu_cores_required": 36 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_ROLLING_BATCH_SIZE": "4", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "4", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1" + }, + "instance_type": "ml.g6e.12xlarge", + "profile": "Default" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1024000, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1" + }, + "instance_type": "ml.g6e.48xlarge" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1539316, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1" + }, + "instance_type": "ml.p5.48xlarge" + } + ], + "hp_eks_override_params_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_sft_fft_override_params_k8s_v1.2.2.json", + "hp_eks_payload_template_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_sft_fft_payload_template_k8s_v1.2.2.yaml", + "instance_count": 1, + "is_subscription_model": false, + "name": "llmft_gpt_oss_20b_seq4k_gpu_sft_fft", + "recipe_file_path": "recipes/fine-tuning/gpt_oss/llmft_gpt_oss_20b_seq4k_gpu_sft_fft.yaml", + "regional_parameters": { + "beta": { + "us-west-2": { + "hp_eks_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-SFT-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + } + }, + "gamma": { + "us-east-1": { + "hp_eks_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-SFT-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-SFT-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + } + }, + "prod": { + "af-south-1": { + "inference": "626614931356.dkr.ecr.af-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-east-1": { + "inference": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-1": { + "hp_eks_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "APN1-ServerlessTraining:FineTuning-SFT-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "ap-northeast-2": { + "inference": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-3": { + "inference": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-1": { + "hp_eks_regional_ecr_uri": "423350936952.dkr.ecr.ap-south-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-2": { + "inference": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-1": { + "hp_eks_regional_ecr_uri": "885852567298.dkr.ecr.ap-southeast-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-2": { + "hp_eks_regional_ecr_uri": "304708117039.dkr.ecr.ap-southeast-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-3": { + "inference": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-4": { + "inference": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-5": { + "inference": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-7": { + "inference": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-central-1": { + "inference": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-west-1": { + "inference": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-north-1": { + "inference": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-northwest-1": { + "inference": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-1": { + "hp_eks_regional_ecr_uri": "391061375763.dkr.ecr.eu-central-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-2": { + "inference": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-north-1": { + "hp_eks_regional_ecr_uri": "963403601044.dkr.ecr.eu-north-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-1": { + "inference": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-2": { + "hp_eks_regional_ecr_uri": "330290781619.dkr.ecr.eu-south-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-1": { + "hp_eks_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "EU-ServerlessTraining:FineTuning-SFT-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "eu-west-2": { + "hp_eks_regional_ecr_uri": "016839105697.dkr.ecr.eu-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-3": { + "inference": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "il-central-1": { + "inference": "780543022126.dkr.ecr.il-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-central-1": { + "inference": "914824155844.dkr.ecr.me-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-south-1": { + "inference": "217643126080.dkr.ecr.me-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "mx-central-1": { + "inference": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "sa-east-1": { + "hp_eks_regional_ecr_uri": "311136344257.dkr.ecr.sa-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-east-1": { + "hp_eks_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-SFT-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "us-east-2": { + "hp_eks_regional_ecr_uri": "556809692997.dkr.ecr.us-east-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-east-1": { + "inference": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-west-1": { + "inference": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-iso-east-1": { + "inference": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isob-east-1": { + "inference": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-east-1": { + "inference": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-south-1": { + "inference": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-1": { + "hp_eks_regional_ecr_uri": "827510180725.dkr.ecr.us-west-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-SFT-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + } + } + }, + "sequence_length": "4K", + "serverless_metering_type": "Token-based", + "smtj_override_params_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_sft_fft_override_params_sm_jobs_v1.2.2.json", + "smtj_recipe_template_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_sft_fft_payload_template_sm_jobs_v1.2.2.yaml", + "supported_instance_types": [ + "ml.p5.48xlarge", + "ml.p5e.48xlarge", + "ml.p5en.48xlarge" + ], + "type": "FineTuning", + "versions": [ + "1.2.2" + ] + }, + { + "customization_technique": "DPO", + "display_name": "GPT-OSS 20B Direct Preference Optimization Full Fine-Tuning on GPU", + "hardware": "GPU", + "hosting_configs": [ + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 196608, + "number_of_accelerator_devices_required": 4, + "number_of_cpu_cores_required": 36 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_ROLLING_BATCH_SIZE": "4", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "4", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1" + }, + "instance_type": "ml.g6e.12xlarge", + "profile": "Default" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1024000, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1" + }, + "instance_type": "ml.g6e.48xlarge" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1539316, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1" + }, + "instance_type": "ml.p5.48xlarge" + } + ], + "hp_eks_override_params_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_dpo_fft_override_params_k8s_v1.3.2.json", + "hp_eks_payload_template_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_dpo_fft_payload_template_k8s_v1.3.2.yaml", + "instance_count": 2, + "is_subscription_model": false, + "name": "llmft_gpt_oss_20b_seq4k_gpu_dpo_fft", + "recipe_file_path": "recipes/fine-tuning/gpt_oss/llmft_gpt_oss_20b_seq4k_gpu_dpo_fft.yaml", + "regional_parameters": { + "beta": { + "us-west-2": { + "hp_eks_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-DPO-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + } + }, + "gamma": { + "us-east-1": { + "hp_eks_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-DPO-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-DPO-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + } + }, + "prod": { + "af-south-1": { + "inference": "626614931356.dkr.ecr.af-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-east-1": { + "inference": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-1": { + "hp_eks_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "APN1-ServerlessTraining:FineTuning-DPO-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "ap-northeast-2": { + "inference": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-3": { + "inference": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-1": { + "hp_eks_regional_ecr_uri": "423350936952.dkr.ecr.ap-south-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-2": { + "inference": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-1": { + "hp_eks_regional_ecr_uri": "885852567298.dkr.ecr.ap-southeast-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-2": { + "hp_eks_regional_ecr_uri": "304708117039.dkr.ecr.ap-southeast-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-3": { + "inference": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-4": { + "inference": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-5": { + "inference": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-7": { + "inference": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-central-1": { + "inference": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-west-1": { + "inference": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-north-1": { + "inference": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-northwest-1": { + "inference": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-1": { + "hp_eks_regional_ecr_uri": "391061375763.dkr.ecr.eu-central-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-2": { + "inference": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-north-1": { + "hp_eks_regional_ecr_uri": "963403601044.dkr.ecr.eu-north-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-1": { + "inference": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-2": { + "hp_eks_regional_ecr_uri": "330290781619.dkr.ecr.eu-south-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-1": { + "hp_eks_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "EU-ServerlessTraining:FineTuning-DPO-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "eu-west-2": { + "hp_eks_regional_ecr_uri": "016839105697.dkr.ecr.eu-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-3": { + "inference": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "il-central-1": { + "inference": "780543022126.dkr.ecr.il-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-central-1": { + "inference": "914824155844.dkr.ecr.me-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-south-1": { + "inference": "217643126080.dkr.ecr.me-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "mx-central-1": { + "inference": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "sa-east-1": { + "hp_eks_regional_ecr_uri": "311136344257.dkr.ecr.sa-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-east-1": { + "hp_eks_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-DPO-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + }, + "us-east-2": { + "hp_eks_regional_ecr_uri": "556809692997.dkr.ecr.us-east-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-east-1": { + "inference": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-west-1": { + "inference": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-iso-east-1": { + "inference": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isob-east-1": { + "inference": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-east-1": { + "inference": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-south-1": { + "inference": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-1": { + "hp_eks_regional_ecr_uri": "827510180725.dkr.ecr.us-west-1.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-DPO-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:llmft-v1.0.0" + } + } + }, + "sequence_length": "4K", + "serverless_metering_type": "Token-based", + "smtj_override_params_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_dpo_fft_override_params_sm_jobs_v1.3.2.json", + "smtj_recipe_template_s3_key": "recipes/llmft_gpt_oss_20b_seq4k_gpu_dpo_fft_payload_template_sm_jobs_v1.3.2.yaml", + "supported_instance_types": [ + "ml.p5.48xlarge", + "ml.p5e.48xlarge", + "ml.p5en.48xlarge" + ], + "type": "FineTuning", + "versions": [ + "1.3.2" + ] + }, + { + "customization_technique": "RLAIF", + "display_name": "GPT OSS 20B GRPO RLAIF Fine-Tuning", + "hardware": "GPU", + "hosting_configs": [ + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 196608, + "number_of_accelerator_devices_required": 4, + "number_of_cpu_cores_required": 36 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_ROLLING_BATCH_SIZE": "4", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "4", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1" + }, + "instance_type": "ml.g6e.12xlarge", + "profile": "Default" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1024000, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1" + }, + "instance_type": "ml.g6e.48xlarge" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1539316, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1" + }, + "instance_type": "ml.p5.48xlarge" + } + ], + "hp_eks_override_params_s3_key": "recipes/verl-grpo-rlaif-gpt-oss-20b-fft_override_params_k8s_v1.2.3.json", + "hp_eks_payload_template_s3_key": "recipes/verl-grpo-rlaif-gpt-oss-20b-fft_payload_template_k8s_v1.2.3.yaml", + "instance_count": 2, + "is_subscription_model": false, + "name": "verl-grpo-rlaif-gpt-oss-20b-fft", + "recipe_file_path": "recipes/fine-tuning/gpt_oss/verl-grpo-rlaif-gpt-oss-20b-fft.yaml", + "regional_parameters": { + "beta": { + "us-west-2": { + "hp_eks_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-RLAIF-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + } + }, + "gamma": { + "us-east-1": { + "hp_eks_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-RLAIF-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-RLAIF-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + } + }, + "prod": { + "af-south-1": { + "inference": "626614931356.dkr.ecr.af-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-east-1": { + "inference": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-1": { + "hp_eks_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "APN1-ServerlessTraining:FineTuning-RLAIF-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "ap-northeast-2": { + "inference": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-3": { + "inference": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-1": { + "hp_eks_regional_ecr_uri": "423350936952.dkr.ecr.ap-south-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-2": { + "inference": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-1": { + "hp_eks_regional_ecr_uri": "885852567298.dkr.ecr.ap-southeast-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-2": { + "hp_eks_regional_ecr_uri": "304708117039.dkr.ecr.ap-southeast-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-3": { + "inference": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-4": { + "inference": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-5": { + "inference": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-7": { + "inference": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-central-1": { + "inference": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-west-1": { + "inference": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-north-1": { + "inference": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-northwest-1": { + "inference": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-1": { + "hp_eks_regional_ecr_uri": "391061375763.dkr.ecr.eu-central-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-2": { + "inference": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-north-1": { + "hp_eks_regional_ecr_uri": "963403601044.dkr.ecr.eu-north-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-1": { + "inference": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-2": { + "hp_eks_regional_ecr_uri": "330290781619.dkr.ecr.eu-south-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-1": { + "hp_eks_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "EU-ServerlessTraining:FineTuning-RLAIF-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "eu-west-2": { + "hp_eks_regional_ecr_uri": "016839105697.dkr.ecr.eu-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-3": { + "inference": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "il-central-1": { + "inference": "780543022126.dkr.ecr.il-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-central-1": { + "inference": "914824155844.dkr.ecr.me-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-south-1": { + "inference": "217643126080.dkr.ecr.me-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "mx-central-1": { + "inference": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "sa-east-1": { + "hp_eks_regional_ecr_uri": "311136344257.dkr.ecr.sa-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-east-1": { + "hp_eks_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-RLAIF-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "us-east-2": { + "hp_eks_regional_ecr_uri": "556809692997.dkr.ecr.us-east-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-east-1": { + "inference": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-west-1": { + "inference": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-iso-east-1": { + "inference": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isob-east-1": { + "inference": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-east-1": { + "inference": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-south-1": { + "inference": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-1": { + "hp_eks_regional_ecr_uri": "827510180725.dkr.ecr.us-west-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-RLAIF-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + } + } + }, + "sequence_length": "4K", + "serverless_metering_type": "Hourly", + "smtj_override_params_s3_key": "recipes/verl-grpo-rlaif-gpt-oss-20b-fft_override_params_sm_jobs_v1.2.3.json", + "smtj_recipe_template_s3_key": "recipes/verl-grpo-rlaif-gpt-oss-20b-fft_payload_template_sm_jobs_v1.2.3.yaml", + "supported_instance_types": [ + "ml.p5.48xlarge" + ], + "type": "FineTuning", + "versions": [ + "1.2.3" + ] + }, + { + "customization_technique": "RLVR", + "display_name": "GPT OSS 20B GRPO RLVR Fine-Tuning", + "hardware": "GPU", + "hosting_configs": [ + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 196608, + "number_of_accelerator_devices_required": 4, + "number_of_cpu_cores_required": 36 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_ROLLING_BATCH_SIZE": "4", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "4", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1" + }, + "instance_type": "ml.g6e.12xlarge", + "profile": "Default" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1024000, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1" + }, + "instance_type": "ml.g6e.48xlarge" + }, + { + "compute_resource_requirements": { + "min_memory_required_in_mb": 1539316, + "number_of_accelerator_devices_required": 8, + "number_of_cpu_cores_required": 100 + }, + "ecr_address": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "environment": { + "OPTION_ASYNC_MODE": "true", + "OPTION_ENABLE_AUTO_TOOL_CHOICE": "true", + "OPTION_ENTRYPOINT": "djl_python.lmi_vllm.vllm_async_service", + "OPTION_MAX_ROLLING_BATCH_SIZE": "8", + "OPTION_ROLLING_BATCH": "disable", + "OPTION_TENSOR_PARALLEL_DEGREE": "8", + "OPTION_TOOL_CALL_PARSER": "openai", + "SAGEMAKER_ENABLE_LOAD_AWARE": "1" + }, + "instance_type": "ml.p5.48xlarge" + } + ], + "hp_eks_override_params_s3_key": "recipes/verl-grpo-rlvr-gpt-oss-20b-fft_override_params_k8s_v1.3.2.json", + "hp_eks_payload_template_s3_key": "recipes/verl-grpo-rlvr-gpt-oss-20b-fft_payload_template_k8s_v1.3.2.yaml", + "instance_count": 2, + "is_subscription_model": false, + "name": "verl-grpo-rlvr-gpt-oss-20b-fft", + "recipe_file_path": "recipes/fine-tuning/gpt_oss/verl-grpo-rlvr-gpt-oss-20b-fft.yaml", + "regional_parameters": { + "beta": { + "us-west-2": { + "hp_eks_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-RLVR-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "300869608763.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + } + }, + "gamma": { + "us-east-1": { + "hp_eks_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-RLVR-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "190594010507.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-RLVR-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "839249767557.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + } + }, + "prod": { + "af-south-1": { + "inference": "626614931356.dkr.ecr.af-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-east-1": { + "inference": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-1": { + "hp_eks_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "APN1-ServerlessTraining:FineTuning-RLVR-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "356859066553.dkr.ecr.ap-northeast-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "ap-northeast-2": { + "inference": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-northeast-3": { + "inference": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-1": { + "hp_eks_regional_ecr_uri": "423350936952.dkr.ecr.ap-south-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-south-2": { + "inference": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-1": { + "hp_eks_regional_ecr_uri": "885852567298.dkr.ecr.ap-southeast-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-2": { + "hp_eks_regional_ecr_uri": "304708117039.dkr.ecr.ap-southeast-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-3": { + "inference": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-4": { + "inference": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-5": { + "inference": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ap-southeast-7": { + "inference": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-central-1": { + "inference": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "ca-west-1": { + "inference": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-north-1": { + "inference": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "cn-northwest-1": { + "inference": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-1": { + "hp_eks_regional_ecr_uri": "391061375763.dkr.ecr.eu-central-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-central-2": { + "inference": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-north-1": { + "hp_eks_regional_ecr_uri": "963403601044.dkr.ecr.eu-north-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-1": { + "inference": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-south-2": { + "hp_eks_regional_ecr_uri": "330290781619.dkr.ecr.eu-south-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-1": { + "hp_eks_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "EU-ServerlessTraining:FineTuning-RLVR-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "942446708630.dkr.ecr.eu-west-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "eu-west-2": { + "hp_eks_regional_ecr_uri": "016839105697.dkr.ecr.eu-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "eu-west-3": { + "inference": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "il-central-1": { + "inference": "780543022126.dkr.ecr.il-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-central-1": { + "inference": "914824155844.dkr.ecr.me-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "me-south-1": { + "inference": "217643126080.dkr.ecr.me-south-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "mx-central-1": { + "inference": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "sa-east-1": { + "hp_eks_regional_ecr_uri": "311136344257.dkr.ecr.sa-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-east-1": { + "hp_eks_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USE1-ServerlessTraining:FineTuning-RLVR-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "327873000638.dkr.ecr.us-east-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + }, + "us-east-2": { + "hp_eks_regional_ecr_uri": "556809692997.dkr.ecr.us-east-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-east-1": { + "inference": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-gov-west-1": { + "inference": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-iso-east-1": { + "inference": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isob-east-1": { + "inference": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-east-1": { + "inference": "303241398832.dkr.ecr.us-isof-east-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-isof-south-1": { + "inference": "454834333376.dkr.ecr.us-isof-south-1.csp.hci.ic.gov/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-1": { + "hp_eks_regional_ecr_uri": "827510180725.dkr.ecr.us-west-1.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-1.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128" + }, + "us-west-2": { + "hp_eks_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-eks", + "inference": "763104351884.dkr.ecr.us-west-2.amazonaws.com/djl-inference:0.35.0-lmi17.0.0-cu128", + "serverless_sku": "USW2-ServerlessTraining:FineTuning-RLVR-FULLRANK-openai-reasoning-gpt-oss-20b", + "smtj_regional_ecr_uri": "920498770698.dkr.ecr.us-west-2.amazonaws.com/hyperpod-recipes:verl-v1.0.0-smtj" + } + } + }, + "sequence_length": "1K", + "serverless_metering_type": "Hourly", + "smtj_override_params_s3_key": "recipes/verl-grpo-rlvr-gpt-oss-20b-fft_override_params_sm_jobs_v1.3.2.json", + "smtj_recipe_template_s3_key": "recipes/verl-grpo-rlvr-gpt-oss-20b-fft_payload_template_sm_jobs_v1.3.2.yaml", + "supported_instance_types": [ + "ml.p5.48xlarge" + ], + "type": "FineTuning", + "versions": [ + "1.3.2" + ] + } + ], + "resource_name_base": "openai-reasoning-gpt-oss-20b", + "supported_inference_instance_types": [ + "ml.g7e.2xlarge", + "ml.p5.48xlarge", + "ml.p5en.48xlarge", + "ml.p5e.48xlarge", + "ml.g6.12xlarge", + "ml.g5.12xlarge", + "ml.g6.24xlarge", + "ml.g5.24xlarge" + ], + "training_artifact_key": "openai-reasoning/openai-reasoning-gpt-oss-20b/artifacts/training/v2.0.0/", + "training_dependencies": [], + "training_supported": false, + "training_vulnerabilities": [], + "training_vulnerable": false, + "url": "https://huggingface.co/openai/gpt-oss-20b", + "validation_supported": true, + "version": "3.38.0" +} \ No newline at end of file diff --git a/sagemaker-serve/tests/unit/servers/data/jumpstart_specs/pytorch-ic-mobilenet-v2.json b/sagemaker-serve/tests/unit/servers/data/jumpstart_specs/pytorch-ic-mobilenet-v2.json new file mode 100644 index 0000000000..16036c5025 --- /dev/null +++ b/sagemaker-serve/tests/unit/servers/data/jumpstart_specs/pytorch-ic-mobilenet-v2.json @@ -0,0 +1,1093 @@ +{ + "default_inference_instance_type": "ml.m5.large", + "default_training_dataset_key": "training-datasets/tf_flowers/", + "default_training_instance_type": "ml.m5.xlarge", + "deploy_kwargs": {}, + "deprecated": false, + "dynamic_container_deployment_supported": false, + "estimator_kwargs": { + "encrypt_inter_container_traffic": true, + "max_run": 360000 + }, + "fine_tuning_supported": true, + "fit_kwargs": {}, + "hosting_artifact_compression_type": "None", + "hosting_artifact_key": "pytorch-ic/pytorch-ic-mobilenet-v2/artifacts/inference/v2.0.0/", + "hosting_artifact_s3_data_type": "S3Prefix", + "hosting_ecr_specs": { + "framework": "pytorch", + "framework_version": "1.10.0", + "py_version": "py38" + }, + "hosting_instance_type_variants": { + "regional_aliases": { + "af-south-1": { + "cpu_ecr_uri_1": "626614931356.dkr.ecr.af-south-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "626614931356.dkr.ecr.af-south-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-east-1": { + "cpu_ecr_uri_1": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-east-2": { + "cpu_ecr_uri_1": "975050140332.dkr.ecr.ap-east-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "975050140332.dkr.ecr.ap-east-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-northeast-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-northeast-2": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-northeast-3": { + "cpu_ecr_uri_1": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-south-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-south-2": { + "alias_ecr_uri_3": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-southeast-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-southeast-2": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-southeast-3": { + "cpu_ecr_uri_1": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-southeast-4": { + "alias_ecr_uri_3": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-southeast-5": { + "cpu_ecr_uri_1": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-southeast-6": { + "cpu_ecr_uri_1": "633930458069.dkr.ecr.ap-southeast-6.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "633930458069.dkr.ecr.ap-southeast-6.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ap-southeast-7": { + "cpu_ecr_uri_1": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ca-central-1": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "ca-west-1": { + "alias_ecr_uri_3": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "cn-north-1": { + "cpu_ecr_uri_1": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/pytorch-inference:1.10.0-gpu-py38" + }, + "cn-northwest-1": { + "cpu_ecr_uri_1": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/pytorch-inference:1.10.0-gpu-py38" + }, + "eu-central-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "eu-central-2": { + "alias_ecr_uri_3": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "eu-north-1": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "eu-south-1": { + "cpu_ecr_uri_1": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "eu-south-2": { + "alias_ecr_uri_3": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "eu-west-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "eu-west-2": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "eu-west-3": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "il-central-1": { + "alias_ecr_uri_3": "780543022126.dkr.ecr.il-central-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "780543022126.dkr.ecr.il-central-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "780543022126.dkr.ecr.il-central-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "me-central-1": { + "cpu_ecr_uri_1": "914824155844.dkr.ecr.me-central-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "914824155844.dkr.ecr.me-central-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "me-south-1": { + "cpu_ecr_uri_1": "217643126080.dkr.ecr.me-south-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "217643126080.dkr.ecr.me-south-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "mx-central-1": { + "cpu_ecr_uri_1": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "sa-east-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "us-east-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "us-east-2": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "us-gov-east-1": { + "cpu_ecr_uri_1": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "us-gov-west-1": { + "cpu_ecr_uri_1": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "us-iso-east-1": { + "cpu_ecr_uri_1": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/pytorch-inference:1.10.0-gpu-py38" + }, + "us-isob-east-1": { + "cpu_ecr_uri_1": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/pytorch-inference:1.10.0-gpu-py38" + }, + "us-west-1": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-west-1.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-west-1.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + }, + "us-west-2": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-inference:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-inference:1.10.0-gpu-py38" + } + }, + "variants": { + "c4": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c5": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c5d": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c5n": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c6i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c6id": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c7i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c8g": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "dl1": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "eia1": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "eia2": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "g4dn": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g5": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g6": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g6e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g7e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "i3en": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "local": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "local_gpu": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "m": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m4": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m5": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m5d": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m6i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m6id": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m7g": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m7gd": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m7i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "p2": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p3": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p3dn": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p4d": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p4de": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p5": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p5e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p5en": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p6": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p6e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "r5": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r5d": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r6i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r6id": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r7g": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r7gd": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r7i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r8g": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "t": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "t2": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "t3": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "trn1": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_3" + } + }, + "trn1n": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_3" + } + }, + "trn2": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_3" + } + } + } + }, + "hosting_prepacked_artifact_key": "pytorch-ic/pytorch-ic-mobilenet-v2/artifacts/inference-prepack/v1.0.0/", + "hosting_prepacked_artifact_version": "1.0.0", + "hosting_script_key": "source-directory-tarballs/pytorch/inference/ic/v2.0.0/sourcedir.tar.gz", + "hosting_use_script_uri": false, + "hyperparameters": [ + { + "default": "True", + "name": "train_only_top_layer", + "options": [ + "True", + "False" + ], + "scope": "algorithm", + "type": "text" + }, + { + "default": 5, + "max": 1000, + "min": 1, + "name": "epochs", + "scope": "algorithm", + "type": "int" + }, + { + "default": 0.001, + "max": 1, + "min": 1e-08, + "name": "learning_rate", + "scope": "algorithm", + "type": "float" + }, + { + "default": 4, + "max": 1024, + "min": 1, + "name": "batch_size", + "scope": "algorithm", + "type": "int" + }, + { + "default": "Auto", + "name": "reinitialize_top_layer", + "options": [ + "Auto", + "True", + "False" + ], + "scope": "algorithm", + "type": "text" + }, + { + "default": "/opt/ml/input/data/code/sourcedir.tar.gz", + "name": "sagemaker_submit_directory", + "scope": "container", + "type": "text" + }, + { + "default": "transfer_learning.py", + "name": "sagemaker_program", + "scope": "container", + "type": "text" + }, + { + "default": "20", + "name": "sagemaker_container_log_level", + "scope": "container", + "type": "text" + } + ], + "incremental_training_supported": true, + "inference_dependencies": [], + "inference_enable_network_isolation": true, + "inference_environment_variables": [ + { + "default": 3600, + "name": "ENDPOINT_SERVER_TIMEOUT", + "required_for_model_class": true, + "scope": "container", + "type": "int" + }, + { + "default": "/opt/ml/model", + "name": "MODEL_CACHE_ROOT", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "20", + "name": "SAGEMAKER_CONTAINER_LOG_LEVEL", + "required_for_model_class": false, + "scope": "container", + "type": "text" + }, + { + "default": "1", + "name": "SAGEMAKER_ENV", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "3600", + "name": "SAGEMAKER_MODEL_SERVER_TIMEOUT", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": 1, + "name": "SAGEMAKER_MODEL_SERVER_WORKERS", + "required_for_model_class": true, + "scope": "container", + "type": "int" + }, + { + "default": "inference.py", + "name": "SAGEMAKER_PROGRAM", + "required_for_model_class": true, + "scope": "container", + "type": "text" + }, + { + "default": "/opt/ml/model/code", + "name": "SAGEMAKER_SUBMIT_DIRECTORY", + "required_for_model_class": false, + "scope": "container", + "type": "text" + } + ], + "inference_vulnerabilities": [], + "inference_vulnerable": false, + "metrics": [ + { + "Name": "pytorch-ic:val-accuracy", + "Regex": "val_accuracy: ([0-9\\.]+)" + } + ], + "min_sdk_version": "2.189.0", + "model_id": "pytorch-ic-mobilenet-v2", + "model_kwargs": {}, + "predictor_specs": { + "default_accept_type": "application/json", + "default_content_type": "application/x-image", + "supported_accept_types": [ + "application/json;verbose", + "application/json" + ], + "supported_content_types": [ + "application/x-image" + ] + }, + "provider": "pytorch", + "resource_name_base": "pt-ic-mobilenet-v2", + "supported_inference_instance_types": [ + "ml.m5.large", + "ml.m5.xlarge", + "ml.c5.xlarge", + "ml.c5.2xlarge", + "ml.m4.xlarge" + ], + "supported_training_instance_types": [ + "ml.m5.xlarge", + "ml.c5.2xlarge", + "ml.m4.xlarge" + ], + "training_artifact_key": "pytorch-training/v2.0.0/train-pytorch-ic-mobilenet-v2.tar.gz", + "training_dependencies": [ + "sagemaker_jumpstart_prepack_script_utilities==1.0.0" + ], + "training_ecr_specs": { + "framework": "pytorch", + "framework_version": "1.10.0", + "py_version": "py38" + }, + "training_enable_network_isolation": true, + "training_instance_type_variants": { + "regional_aliases": { + "af-south-1": { + "cpu_ecr_uri_1": "626614931356.dkr.ecr.af-south-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "626614931356.dkr.ecr.af-south-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-east-1": { + "cpu_ecr_uri_1": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "871362719292.dkr.ecr.ap-east-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-east-2": { + "cpu_ecr_uri_1": "975050140332.dkr.ecr.ap-east-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "975050140332.dkr.ecr.ap-east-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-northeast-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-northeast-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-northeast-2": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-northeast-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-northeast-3": { + "cpu_ecr_uri_1": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "364406365360.dkr.ecr.ap-northeast-3.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-south-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-south-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-south-2": { + "alias_ecr_uri_3": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "772153158452.dkr.ecr.ap-south-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-southeast-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-southeast-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-southeast-2": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ap-southeast-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-southeast-3": { + "cpu_ecr_uri_1": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "907027046896.dkr.ecr.ap-southeast-3.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-southeast-4": { + "alias_ecr_uri_3": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "457447274322.dkr.ecr.ap-southeast-4.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-southeast-5": { + "cpu_ecr_uri_1": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "550225433462.dkr.ecr.ap-southeast-5.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-southeast-6": { + "cpu_ecr_uri_1": "633930458069.dkr.ecr.ap-southeast-6.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "633930458069.dkr.ecr.ap-southeast-6.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ap-southeast-7": { + "cpu_ecr_uri_1": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "590183813437.dkr.ecr.ap-southeast-7.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ca-central-1": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.ca-central-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "ca-west-1": { + "alias_ecr_uri_3": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "204538143572.dkr.ecr.ca-west-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "cn-north-1": { + "cpu_ecr_uri_1": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "727897471807.dkr.ecr.cn-north-1.amazonaws.com.cn/pytorch-training:1.10.0-gpu-py38" + }, + "cn-northwest-1": { + "cpu_ecr_uri_1": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "727897471807.dkr.ecr.cn-northwest-1.amazonaws.com.cn/pytorch-training:1.10.0-gpu-py38" + }, + "eu-central-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-central-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "eu-central-2": { + "alias_ecr_uri_3": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "380420809688.dkr.ecr.eu-central-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "eu-north-1": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-north-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "eu-south-1": { + "cpu_ecr_uri_1": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "692866216735.dkr.ecr.eu-south-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "eu-south-2": { + "alias_ecr_uri_3": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "503227376785.dkr.ecr.eu-south-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "eu-west-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "eu-west-2": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "eu-west-3": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.eu-west-3.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "il-central-1": { + "alias_ecr_uri_3": "780543022126.dkr.ecr.il-central-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "780543022126.dkr.ecr.il-central-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "780543022126.dkr.ecr.il-central-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "me-central-1": { + "cpu_ecr_uri_1": "914824155844.dkr.ecr.me-central-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "914824155844.dkr.ecr.me-central-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "me-south-1": { + "cpu_ecr_uri_1": "217643126080.dkr.ecr.me-south-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "217643126080.dkr.ecr.me-south-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "mx-central-1": { + "cpu_ecr_uri_1": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "637423239942.dkr.ecr.mx-central-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "sa-east-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.sa-east-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "us-east-1": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-east-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "us-east-2": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-east-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "us-gov-east-1": { + "cpu_ecr_uri_1": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "446045086412.dkr.ecr.us-gov-east-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "us-gov-west-1": { + "cpu_ecr_uri_1": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "442386744353.dkr.ecr.us-gov-west-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "us-iso-east-1": { + "cpu_ecr_uri_1": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "886529160074.dkr.ecr.us-iso-east-1.c2s.ic.gov/pytorch-training:1.10.0-gpu-py38" + }, + "us-isob-east-1": { + "cpu_ecr_uri_1": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "094389454867.dkr.ecr.us-isob-east-1.sc2s.sgov.gov/pytorch-training:1.10.0-gpu-py38" + }, + "us-west-1": { + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-west-1.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-west-1.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + }, + "us-west-2": { + "alias_ecr_uri_3": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training-neuron:1.11.0-neuron-py38-sdk2.4.0-ubuntu20.04", + "cpu_ecr_uri_1": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training:1.10.0-cpu-py38", + "gpu_ecr_uri_2": "763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training:1.10.0-gpu-py38" + } + }, + "variants": { + "c4": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c5": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c5d": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c5n": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c6i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c6id": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c7i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "c8g": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "dl1": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "eia1": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "eia2": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "g4dn": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g5": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g6": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g6e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "g7e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "i3en": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "local": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "local_gpu": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "m": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m4": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m5": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m5d": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m6i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m6id": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m7g": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m7gd": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "m7i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "p2": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p3": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p3dn": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p4d": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p4de": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p5": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p5e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p5en": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p6": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "p6e": { + "regional_properties": { + "image_uri": "$gpu_ecr_uri_2" + } + }, + "r5": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r5d": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r6i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r6id": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r7g": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r7gd": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r7i": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "r8g": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "t": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "t2": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "t3": { + "regional_properties": { + "image_uri": "$cpu_ecr_uri_1" + } + }, + "trn1": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_3" + } + }, + "trn1n": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_3" + } + }, + "trn2": { + "regional_properties": { + "image_uri": "$alias_ecr_uri_3" + } + } + } + }, + "training_prepacked_script_key": "source-directory-tarballs/pytorch/transfer_learning/ic/prepack/v1.1.0/sourcedir.tar.gz", + "training_prepacked_script_version": "1.1.0", + "training_script_key": "source-directory-tarballs/pytorch/transfer_learning/ic/v2.3.0/sourcedir.tar.gz", + "training_supported": true, + "training_vulnerabilities": [], + "training_vulnerable": false, + "url": "https://pytorch.org/hub/pytorch_vision_mobilenet_v2/", + "validation_supported": false, + "version": "3.0.25" +} \ No newline at end of file diff --git a/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py b/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py index 8a6471133f..f191335ee3 100644 --- a/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py +++ b/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py @@ -1007,202 +1007,234 @@ def test_build_sagemaker_endpoint_djl(self, mock_create, mock_prepare, mock_init class TestJumpStartAdditionalModelDataSourcesUserFlow(unittest.TestCase): - """User-flow tests for additional model data sources, driven by raw spec JSON. + """User-flow tests for additional model data sources, driven by REAL specs. - Unlike the Mock-based tests above, these tests exercise the code the way a - user calls it: + Each test calls the SDK exactly the way a customer does: - builder = ModelBuilder(model="", ...) + builder = ModelBuilder(model="openai-reasoning-gpt-oss-20b", + instance_type="ml.g7e.2xlarge", ...) builder.accept_eula = True # the documented EULA knob builder.build() - Only the JSON-fetch boundary is mocked (the spec and manifest accessors, - fed from tests/unit/servers/data/jumpstart_specs.json) plus the AWS - boundary (_prepare_for_mode / _create_model). Everything in between -- - JumpStartModelSpecs parsing, the factory shaping in get_init_kwargs - (bucket resolution, PascalCase conversion), JumpStart model-id detection, - and _build_for_jumpstart's EULA handling -- runs for real. - - The fixture file holds one raw snake_case spec per use case (the shape - published to the JumpStart content bucket): public model without - additional sources, public model with a public source, gated model with a - gated source, and a public model with mixed public+gated sources. + The spec fixtures under tests/unit/servers/data/jumpstart_specs/ are real, + unmodified specs captured from the production JumpStart content bucket + (jumpstart-cache-prod-us-west-2): + + - pytorch-ic-mobilenet-v2: public model, no additional data sources. + - openai-reasoning-gpt-oss-20b: public model whose default (lmi-optimized) + config carries an ungated EAGLE speculative-decoding source -- the model + that surfaced the propagation bug this suite guards. + - meta-textgeneration-llama-3-1-70b: gated model whose lmi-optimized + config carries a GATED draft_model source + (hosting_eula_key=fmhMetadata/eula/llama3_2Eula.txt). + + Only two seams are patched, both once in setUp (the same seams the legacy + master-v2 JumpStartModel tests patched): + + 1. The spec-fetch boundary: JumpStartModelsAccessor.get_model_specs routes + by model id into the captured spec files (master-v2's + PROTOTYPICAL_MODEL_SPECS_DICT pattern), and _get_manifest serves the + captured manifest headers. + 2. The AWS boundary: _prepare_for_mode / _create_model and a mock session. + + Everything in between runs for real: JumpStart model-id detection, + JumpStartModelSpecs parsing, inference-config resolution, the + get_init_kwargs factory (image URI resolution, content-bucket injection, + PascalCase shaping), and _build_for_jumpstart's EULA translation. """ - SPEC_FIXTURE_PATH = Path(__file__).parent / "data" / "jumpstart_specs.json" - MODEL_ID = "pytorch-ic-mobilenet-v2" + SPEC_DIR = Path(__file__).parent / "data" / "jumpstart_specs" - EXPECTED_PUBLIC_SOURCE = { - "ChannelName": "draft_model", + ROLE_ARN = "arn:aws:iam::123456789012:role/SageMakerRole" + + # The real EAGLE source published in gpt-oss's default (lmi-optimized) + # config, after factory shaping: content bucket injected, PascalCase, and + # the null HostingEulaKey stripped by _build_for_jumpstart. + GPT_OSS_EAGLE_SOURCE = { + "ChannelName": "eagle", "S3DataSource": { "CompressionType": "None", "S3DataType": "S3Prefix", - "S3Uri": "s3://jumpstart-cache-prod-us-west-2/key/to/draft/model/", + "S3Uri": "s3://jumpstart-cache-prod-us-west-2/lmi-eagle-heads/models/gpt-oss-20b-p-eagle/", }, } - EXPECTED_GATED_SOURCE_ACCEPTED = { - "ChannelName": "gated_draft_model", + + # The real gated llama-3.2-1b draft model published in llama-3-1-70b's + # lmi-optimized config: private content bucket, EULA acceptance folded + # into ModelAccessConfig, HostingEulaKey stripped. + LLAMA_GATED_DRAFT_SOURCE_ACCEPTED = { + "ChannelName": "draft_model", "S3DataSource": { "CompressionType": "None", "S3DataType": "S3Prefix", - "S3Uri": "s3://jumpstart-private-cache-prod-us-west-2/key/to/gated/draft/", + "S3Uri": ( + "s3://jumpstart-private-cache-prod-us-west-2/meta-textgeneration/" + "meta-textgeneration-llama-3-2-1b/artifacts/inference-prepack/v1.0.0/" + ), "ModelAccessConfig": {"AcceptEula": True}, }, } - @classmethod - def setUpClass(cls): - with open(cls.SPEC_FIXTURE_PATH) as f: - cls.spec_fixture = json.load(f) - - def _spec_json(self, variant): - """Returns the raw spec JSON for a named fixture variant.""" - spec = json.loads(json.dumps(self.spec_fixture["base_spec"])) - spec.update(self.spec_fixture["variants"][variant]) - return spec - - def _make_builder(self, variant, accept_eula=None): - """Builds a real ModelBuilder the way a user would, with the JSON-fetch - boundary patched to serve the fixture variant.""" + def setUp(self): from sagemaker.core.jumpstart.accessors import JumpStartModelsAccessor from sagemaker.core.jumpstart.types import JumpStartModelHeader, JumpStartModelSpecs from sagemaker.serve.model_builder import ModelBuilder - spec_json = self._spec_json(variant) + with open(self.SPEC_DIR / "manifest.json") as f: + manifest = [JumpStartModelHeader(header) for header in json.load(f)] - session = Mock() - session.boto_region_name = "us-west-2" - session.get_caller_identity_arn = Mock( - return_value="arn:aws:iam::123456789012:role/SageMakerRole" - ) - session.sagemaker_config = {} - session.config = None + spec_dir = self.SPEC_DIR - header = JumpStartModelHeader( - { - "model_id": self.MODEL_ID, - "version": "1.0.0", - "min_version": "2.49.0", - "spec_key": f"community_models_specs/{self.MODEL_ID}/specs_v1.0.0.json", - } - ) + def get_captured_model_specs(*args, **kwargs): + """Routes spec lookups by model id into the captured real spec + files, like master-v2's PROTOTYPICAL_MODEL_SPECS_DICT loaders.""" + model_id = kwargs.get("model_id") or args[1] + with open(spec_dir / f"{model_id}.json") as f: + return JumpStartModelSpecs(json.load(f)) patchers = [ patch.object( - JumpStartModelsAccessor, - "get_model_specs", - side_effect=lambda *a, **kw: JumpStartModelSpecs( - json.loads(json.dumps(spec_json)) - ), + JumpStartModelsAccessor, "get_model_specs", side_effect=get_captured_model_specs ), - patch.object(JumpStartModelsAccessor, "_get_manifest", return_value=[header]), + patch.object(JumpStartModelsAccessor, "_get_manifest", return_value=manifest), patch.object(ModelBuilder, "_prepare_for_mode", return_value=None), patch.object(ModelBuilder, "_create_model", return_value=Mock()), ] - for p in patchers: - p.start() - self.addCleanup(p.stop) + for patcher in patchers: + patcher.start() + self.addCleanup(patcher.stop) + + session = Mock() + session.boto_region_name = "us-west-2" + session.get_caller_identity_arn = Mock(return_value=self.ROLE_ARN) + session.sagemaker_config = {} + session.config = None + self.session = session + + def _model_builder(self, model_id, instance_type=None): + """Constructs a ModelBuilder the way a customer does.""" + from sagemaker.serve.model_builder import ModelBuilder - builder = ModelBuilder( - model=self.MODEL_ID, + return ModelBuilder( + model=model_id, mode=Mode.SAGEMAKER_ENDPOINT, - role_arn="arn:aws:iam::123456789012:role/SageMakerRole", - sagemaker_session=session, + role_arn=self.ROLE_ARN, + sagemaker_session=self.session, + instance_type=instance_type, ) - if accept_eula is not None: - builder.accept_eula = accept_eula - return builder + + # ------------------------------------------------------------------ + # pytorch-ic-mobilenet-v2: public model, no additional data sources + # ------------------------------------------------------------------ def test_public_model_without_additional_sources(self): - """A spec with no additional data sources builds cleanly and sets none.""" - builder = self._make_builder("public_model_no_additional_sources") + """The base case: a model with no additional data sources builds + cleanly and never sets the attribute.""" + builder = self._model_builder("pytorch-ic-mobilenet-v2") builder.build() self.assertIsNone(getattr(builder, "additional_model_data_sources", None)) - def test_public_additional_source_builds_without_accept_eula(self): - """A public (ungated) additional source deploys without the user ever - touching accept_eula; the internal HostingEulaKey is stripped.""" - builder = self._make_builder("public_model_public_additional_sources") + # ------------------------------------------------------------------ + # openai-reasoning-gpt-oss-20b: ungated EAGLE source in default config + # ------------------------------------------------------------------ + + def test_gpt_oss_eagle_source_propagates_without_accept_eula(self): + """The bug this suite guards: gpt-oss's default config ships an + ungated EAGLE draft model. A customer deploys it without ever touching + accept_eula, and the source must reach CreateModel.""" + builder = self._model_builder( + "openai-reasoning-gpt-oss-20b", instance_type="ml.g7e.2xlarge" + ) builder.build() self.assertEqual( - builder.additional_model_data_sources, [self.EXPECTED_PUBLIC_SOURCE] + builder.additional_model_data_sources, [self.GPT_OSS_EAGLE_SOURCE] ) - def test_public_additional_source_unaffected_by_accept_eula(self): - """Passing accept_eula=True for a public source is harmless: no - ModelAccessConfig is added.""" - builder = self._make_builder( - "public_model_public_additional_sources", accept_eula=True + def test_gpt_oss_eagle_source_unaffected_by_accept_eula(self): + """Setting accept_eula=True for an ungated source is harmless: no + ModelAccessConfig is added to it.""" + builder = self._model_builder( + "openai-reasoning-gpt-oss-20b", instance_type="ml.g7e.2xlarge" ) + builder.accept_eula = True builder.build() self.assertEqual( - builder.additional_model_data_sources, [self.EXPECTED_PUBLIC_SOURCE] + builder.additional_model_data_sources, [self.GPT_OSS_EAGLE_SOURCE] ) - def test_gated_additional_source_with_accept_eula_builds(self): - """User sets accept_eula=True on the builder: the gated source deploys - with the acceptance recorded in its ModelAccessConfig.""" - builder = self._make_builder( - "gated_model_gated_additional_sources", accept_eula=True + # ------------------------------------------------------------------ + # meta-textgeneration-llama-3-1-70b: gated draft_model source in the + # lmi-optimized config (selected via the public set_deployment_config API) + # ------------------------------------------------------------------ + + def test_llama_gated_draft_source_with_accept_eula_builds(self): + """A customer selects the lmi-optimized config and accepts the EULA: + the gated draft model deploys with acceptance recorded in its + ModelAccessConfig.""" + builder = self._model_builder( + "meta-textgeneration-llama-3-1-70b", instance_type="ml.p4d.24xlarge" + ) + builder.set_deployment_config( + config_name="lmi-optimized", instance_type="ml.p4d.24xlarge" ) + builder.accept_eula = True builder.build() self.assertEqual( - builder.additional_model_data_sources, [self.EXPECTED_GATED_SOURCE_ACCEPTED] + builder.additional_model_data_sources, + [self.LLAMA_GATED_DRAFT_SOURCE_ACCEPTED], ) - def test_gated_additional_source_without_accept_eula_raises(self): - """User never sets accept_eula: build() fails with the actionable - EULA error instead of silently deploying the gated source.""" - builder = self._make_builder("gated_model_gated_additional_sources") + def test_llama_gated_draft_source_without_accept_eula_raises(self): + """A customer selects the gated draft config but never accepts the + EULA: build() fails with the actionable error instead of silently + deploying the gated channel.""" + builder = self._model_builder( + "meta-textgeneration-llama-3-1-70b", instance_type="ml.p4d.24xlarge" + ) + builder.set_deployment_config( + config_name="lmi-optimized", instance_type="ml.p4d.24xlarge" + ) with self.assertRaises(ValueError) as ctx: builder.build() self.assertIn("accept_eula must be set to True", str(ctx.exception)) - def test_gated_additional_source_with_accept_eula_false_raises(self): - """User explicitly sets accept_eula=False: build() fails the same way.""" - builder = self._make_builder( - "gated_model_gated_additional_sources", accept_eula=False + def test_llama_gated_draft_source_with_accept_eula_false_raises(self): + """A customer explicitly sets accept_eula=False: build() fails the + same way.""" + builder = self._model_builder( + "meta-textgeneration-llama-3-1-70b", instance_type="ml.p4d.24xlarge" ) + builder.set_deployment_config( + config_name="lmi-optimized", instance_type="ml.p4d.24xlarge" + ) + builder.accept_eula = False with self.assertRaises(ValueError) as ctx: builder.build() self.assertIn("accept_eula must be set to True", str(ctx.exception)) - def test_mixed_additional_sources_with_accept_eula(self): - """Public + gated sources together with accept_eula=True: the public - source passes through untouched, the gated source gets the - ModelAccessConfig.""" - builder = self._make_builder( - "public_model_mixed_additional_sources", accept_eula=True + def test_llama_default_config_has_no_additional_sources(self): + """The same gated model on its default (lmi) config carries no + additional sources: builds without accept_eula, nothing propagated.""" + builder = self._model_builder( + "meta-textgeneration-llama-3-1-70b", instance_type="ml.p4d.24xlarge" ) + builder.accept_eula = True builder.build() - self.assertEqual( - builder.additional_model_data_sources, - [self.EXPECTED_PUBLIC_SOURCE, self.EXPECTED_GATED_SOURCE_ACCEPTED], - ) - - def test_mixed_additional_sources_without_accept_eula_raises(self): - """Public + gated sources together without accept_eula: the gated - source blocks the build.""" - builder = self._make_builder("public_model_mixed_additional_sources") - - with self.assertRaises(ValueError) as ctx: - builder.build() - - self.assertIn("accept_eula must be set to True", str(ctx.exception)) + self.assertIsNone(getattr(builder, "additional_model_data_sources", None)) class TestDeployWrappers(unittest.TestCase): From 09ee3e765795718c0d21fcc239eae358447d64d7 Mon Sep 17 00:00:00 2001 From: Eli Davidson Date: Tue, 4 Aug 2026 15:31:59 +0000 Subject: [PATCH 5/5] test(serve): assert the CreateModel request shape via the single-call API Restructure the additional-model-data-source tests on three axes: 1. Test names describe the case under test (gated source with accepted EULA sends ModelAccessConfig, unselected config sources do not leak, etc.), not the fixture model. 2. Each test drives the documented single-call customer API: ModelBuilder.from_jumpstart_config(JumpStartConfig(model_id, accept_eula, inference_config_name), compute=Compute(...)) followed by build(). No attribute pokes and no separate config-selection call. 3. Assertions target the CreateModel request itself: _create_model is no longer patched, so the real _create_sagemaker_model and _prepare_container_def_base run, and every test inspects the container_defs captured from sagemaker_session.create_model. This covers the full downstream impact: AdditionalModelDataSources presence/absence/shape, ModelAccessConfig on both the gated additional source and the primary ModelDataSource, and create_model never being called when the EULA gate raises. --- .../servers/test_model_builder_servers.py | 227 +++++++++--------- 1 file changed, 120 insertions(+), 107 deletions(-) diff --git a/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py b/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py index f191335ee3..cb49abaaf8 100644 --- a/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py +++ b/sagemaker-serve/tests/unit/servers/test_model_builder_servers.py @@ -1007,15 +1007,23 @@ def test_build_sagemaker_endpoint_djl(self, mock_create, mock_prepare, mock_init class TestJumpStartAdditionalModelDataSourcesUserFlow(unittest.TestCase): - """User-flow tests for additional model data sources, driven by REAL specs. + """End-to-end tests for additional model data sources, asserting the + CreateModel request shape. - Each test calls the SDK exactly the way a customer does: + Each test uses the documented single-call customer API: - builder = ModelBuilder(model="openai-reasoning-gpt-oss-20b", - instance_type="ml.g7e.2xlarge", ...) - builder.accept_eula = True # the documented EULA knob + builder = ModelBuilder.from_jumpstart_config( + jumpstart_config=JumpStartConfig( + model_id="...", accept_eula=True, inference_config_name="..."), + compute=Compute(instance_type="..."), ...) builder.build() + and asserts on what the SDK sends to the CreateModel API -- the + ``container_defs`` captured from ``sagemaker_session.create_model`` -- not + on builder internals. This is the request boto receives, so the assertions + cover the full downstream impact: primary container image, model data + source, and AdditionalModelDataSources. + The spec fixtures under tests/unit/servers/data/jumpstart_specs/ are real, unmodified specs captured from the production JumpStart content bucket (jumpstart-cache-prod-us-west-2): @@ -1028,19 +1036,22 @@ class TestJumpStartAdditionalModelDataSourcesUserFlow(unittest.TestCase): config carries a GATED draft_model source (hosting_eula_key=fmhMetadata/eula/llama3_2Eula.txt). - Only two seams are patched, both once in setUp (the same seams the legacy - master-v2 JumpStartModel tests patched): + Patching is centralized in setUp on the same seams the legacy master-v2 + JumpStartModel tests patched: 1. The spec-fetch boundary: JumpStartModelsAccessor.get_model_specs routes by model id into the captured spec files (master-v2's - PROTOTYPICAL_MODEL_SPECS_DICT pattern), and _get_manifest serves the + PROTOTYPICAL_MODEL_SPECS_DICT pattern); _get_manifest serves the captured manifest headers. - 2. The AWS boundary: _prepare_for_mode / _create_model and a mock session. + 2. The AWS boundary: a mock session (whose create_model call is the + assertion target), IAM role validation, artifact staging + (_prepare_for_mode), and the post-create Model.get describe call. Everything in between runs for real: JumpStart model-id detection, JumpStartModelSpecs parsing, inference-config resolution, the get_init_kwargs factory (image URI resolution, content-bucket injection, - PascalCase shaping), and _build_for_jumpstart's EULA translation. + PascalCase shaping), _build_for_jumpstart's EULA translation, and + _prepare_container_def_base assembling the CreateModel request. """ SPEC_DIR = Path(__file__).parent / "data" / "jumpstart_specs" @@ -1048,9 +1059,9 @@ class TestJumpStartAdditionalModelDataSourcesUserFlow(unittest.TestCase): ROLE_ARN = "arn:aws:iam::123456789012:role/SageMakerRole" # The real EAGLE source published in gpt-oss's default (lmi-optimized) - # config, after factory shaping: content bucket injected, PascalCase, and - # the null HostingEulaKey stripped by _build_for_jumpstart. - GPT_OSS_EAGLE_SOURCE = { + # config, as it must appear in the CreateModel request: content bucket + # injected, PascalCase, no HostingEulaKey (the API rejects it). + UNGATED_EAGLE_SOURCE = { "ChannelName": "eagle", "S3DataSource": { "CompressionType": "None", @@ -1060,9 +1071,10 @@ class TestJumpStartAdditionalModelDataSourcesUserFlow(unittest.TestCase): } # The real gated llama-3.2-1b draft model published in llama-3-1-70b's - # lmi-optimized config: private content bucket, EULA acceptance folded - # into ModelAccessConfig, HostingEulaKey stripped. - LLAMA_GATED_DRAFT_SOURCE_ACCEPTED = { + # lmi-optimized config, as it must appear in the CreateModel request: + # private content bucket, EULA acceptance folded into ModelAccessConfig, + # no HostingEulaKey. + GATED_DRAFT_SOURCE_ACCEPTED = { "ChannelName": "draft_model", "S3DataSource": { "CompressionType": "None", @@ -1076,6 +1088,7 @@ class TestJumpStartAdditionalModelDataSourcesUserFlow(unittest.TestCase): } def setUp(self): + import sagemaker.serve.model_builder as model_builder_module from sagemaker.core.jumpstart.accessors import JumpStartModelsAccessor from sagemaker.core.jumpstart.types import JumpStartModelHeader, JumpStartModelSpecs from sagemaker.serve.model_builder import ModelBuilder @@ -1098,7 +1111,10 @@ def get_captured_model_specs(*args, **kwargs): ), patch.object(JumpStartModelsAccessor, "_get_manifest", return_value=manifest), patch.object(ModelBuilder, "_prepare_for_mode", return_value=None), - patch.object(ModelBuilder, "_create_model", return_value=Mock()), + patch.object( + model_builder_module, "resolve_and_validate_role", return_value=self.ROLE_ARN + ), + patch.object(model_builder_module.Model, "get", return_value=Mock()), ] for patcher in patchers: patcher.start() @@ -1109,132 +1125,129 @@ def get_captured_model_specs(*args, **kwargs): session.get_caller_identity_arn = Mock(return_value=self.ROLE_ARN) session.sagemaker_config = {} session.config = None + session.settings.include_jumpstart_tags = False + session.default_bucket = Mock(return_value="sagemaker-us-west-2-123456789012") + session.default_bucket_prefix = None + session.local_mode = False self.session = session - def _model_builder(self, model_id, instance_type=None): - """Constructs a ModelBuilder the way a customer does.""" + def _build(self, model_id, accept_eula=None, inference_config_name=None, instance_type=None): + """The customer flow, verbatim: one from_jumpstart_config call carrying + the model id, EULA decision, and config selection, then build().""" + from sagemaker.core.jumpstart.configs import JumpStartConfig + from sagemaker.core.training.configs import Compute from sagemaker.serve.model_builder import ModelBuilder - return ModelBuilder( - model=model_id, - mode=Mode.SAGEMAKER_ENDPOINT, + builder = ModelBuilder.from_jumpstart_config( + jumpstart_config=JumpStartConfig( + model_id=model_id, + accept_eula=accept_eula, + inference_config_name=inference_config_name, + ), role_arn=self.ROLE_ARN, + compute=Compute(instance_type=instance_type) if instance_type else None, sagemaker_session=self.session, - instance_type=instance_type, ) - - # ------------------------------------------------------------------ - # pytorch-ic-mobilenet-v2: public model, no additional data sources - # ------------------------------------------------------------------ - - def test_public_model_without_additional_sources(self): - """The base case: a model with no additional data sources builds - cleanly and never sets the attribute.""" - builder = self._model_builder("pytorch-ic-mobilenet-v2") - builder.build() - self.assertIsNone(getattr(builder, "additional_model_data_sources", None)) + def _create_model_container_def(self): + """The container definition sent to the CreateModel API.""" + self.session.create_model.assert_called_once() + return self.session.create_model.call_args.kwargs["container_defs"] - # ------------------------------------------------------------------ - # openai-reasoning-gpt-oss-20b: ungated EAGLE source in default config - # ------------------------------------------------------------------ + def test_model_without_additional_sources_sends_none_to_create_model(self): + """Base case: the CreateModel request carries no + AdditionalModelDataSources field at all.""" + self._build("pytorch-ic-mobilenet-v2") - def test_gpt_oss_eagle_source_propagates_without_accept_eula(self): - """The bug this suite guards: gpt-oss's default config ships an - ungated EAGLE draft model. A customer deploys it without ever touching - accept_eula, and the source must reach CreateModel.""" - builder = self._model_builder( - "openai-reasoning-gpt-oss-20b", instance_type="ml.g7e.2xlarge" - ) + container_def = self._create_model_container_def() + self.assertNotIn("AdditionalModelDataSources", container_def) - builder.build() + def test_ungated_additional_source_reaches_create_model_without_eula(self): + """The bug this suite guards: an ungated speculative-decoding source + in the model's default config must reach CreateModel even though the + customer never touches accept_eula.""" + self._build("openai-reasoning-gpt-oss-20b", instance_type="ml.g7e.2xlarge") + container_def = self._create_model_container_def() self.assertEqual( - builder.additional_model_data_sources, [self.GPT_OSS_EAGLE_SOURCE] + container_def["AdditionalModelDataSources"], [self.UNGATED_EAGLE_SOURCE] ) - def test_gpt_oss_eagle_source_unaffected_by_accept_eula(self): - """Setting accept_eula=True for an ungated source is harmless: no - ModelAccessConfig is added to it.""" - builder = self._model_builder( - "openai-reasoning-gpt-oss-20b", instance_type="ml.g7e.2xlarge" + def test_ungated_additional_source_gets_no_model_access_config(self): + """accept_eula=True must not add a ModelAccessConfig to an ungated + additional source (it applies to the primary model only).""" + self._build( + "openai-reasoning-gpt-oss-20b", accept_eula=True, instance_type="ml.g7e.2xlarge" ) - builder.accept_eula = True - - builder.build() + container_def = self._create_model_container_def() self.assertEqual( - builder.additional_model_data_sources, [self.GPT_OSS_EAGLE_SOURCE] - ) - - # ------------------------------------------------------------------ - # meta-textgeneration-llama-3-1-70b: gated draft_model source in the - # lmi-optimized config (selected via the public set_deployment_config API) - # ------------------------------------------------------------------ - - def test_llama_gated_draft_source_with_accept_eula_builds(self): - """A customer selects the lmi-optimized config and accepts the EULA: - the gated draft model deploys with acceptance recorded in its - ModelAccessConfig.""" - builder = self._model_builder( - "meta-textgeneration-llama-3-1-70b", instance_type="ml.p4d.24xlarge" + container_def["AdditionalModelDataSources"], [self.UNGATED_EAGLE_SOURCE] ) - builder.set_deployment_config( - config_name="lmi-optimized", instance_type="ml.p4d.24xlarge" + # The single accept_eula knob still covers the primary (base) model. + self.assertEqual( + container_def["ModelDataSource"]["S3DataSource"]["ModelAccessConfig"], + {"AcceptEula": True}, ) - builder.accept_eula = True - - builder.build() - self.assertEqual( - builder.additional_model_data_sources, - [self.LLAMA_GATED_DRAFT_SOURCE_ACCEPTED], + def test_gated_additional_source_with_accepted_eula_sends_model_access_config(self): + """Accepting the EULA folds ModelAccessConfig into the gated + additional source AND the primary model in the CreateModel request.""" + self._build( + "meta-textgeneration-llama-3-1-70b", + accept_eula=True, + inference_config_name="lmi-optimized", + instance_type="ml.p4d.24xlarge", ) - def test_llama_gated_draft_source_without_accept_eula_raises(self): - """A customer selects the gated draft config but never accepts the - EULA: build() fails with the actionable error instead of silently - deploying the gated channel.""" - builder = self._model_builder( - "meta-textgeneration-llama-3-1-70b", instance_type="ml.p4d.24xlarge" + container_def = self._create_model_container_def() + self.assertEqual( + container_def["AdditionalModelDataSources"], [self.GATED_DRAFT_SOURCE_ACCEPTED] ) - builder.set_deployment_config( - config_name="lmi-optimized", instance_type="ml.p4d.24xlarge" + self.assertEqual( + container_def["ModelDataSource"]["S3DataSource"]["ModelAccessConfig"], + {"AcceptEula": True}, ) + def test_gated_additional_source_without_eula_raises_and_never_calls_create_model(self): + """No EULA acceptance: build() fails with the actionable error and no + CreateModel call is ever made.""" with self.assertRaises(ValueError) as ctx: - builder.build() + self._build( + "meta-textgeneration-llama-3-1-70b", + inference_config_name="lmi-optimized", + instance_type="ml.p4d.24xlarge", + ) self.assertIn("accept_eula must be set to True", str(ctx.exception)) + self.session.create_model.assert_not_called() - def test_llama_gated_draft_source_with_accept_eula_false_raises(self): - """A customer explicitly sets accept_eula=False: build() fails the - same way.""" - builder = self._model_builder( - "meta-textgeneration-llama-3-1-70b", instance_type="ml.p4d.24xlarge" - ) - builder.set_deployment_config( - config_name="lmi-optimized", instance_type="ml.p4d.24xlarge" - ) - builder.accept_eula = False - + def test_gated_additional_source_with_rejected_eula_raises_and_never_calls_create_model(self): + """Explicit accept_eula=False fails the same way.""" with self.assertRaises(ValueError) as ctx: - builder.build() + self._build( + "meta-textgeneration-llama-3-1-70b", + accept_eula=False, + inference_config_name="lmi-optimized", + instance_type="ml.p4d.24xlarge", + ) self.assertIn("accept_eula must be set to True", str(ctx.exception)) - - def test_llama_default_config_has_no_additional_sources(self): - """The same gated model on its default (lmi) config carries no - additional sources: builds without accept_eula, nothing propagated.""" - builder = self._model_builder( - "meta-textgeneration-llama-3-1-70b", instance_type="ml.p4d.24xlarge" + self.session.create_model.assert_not_called() + + def test_unselected_config_sources_do_not_leak_into_create_model(self): + """Config resolution gates which sources apply: the same gated model on + its default (lmi) config has no additional sources, so the CreateModel + request must not carry the lmi-optimized config's gated draft model.""" + self._build( + "meta-textgeneration-llama-3-1-70b", + accept_eula=True, + instance_type="ml.p4d.24xlarge", ) - builder.accept_eula = True - - builder.build() - self.assertIsNone(getattr(builder, "additional_model_data_sources", None)) + container_def = self._create_model_container_def() + self.assertNotIn("AdditionalModelDataSources", container_def) class TestDeployWrappers(unittest.TestCase):