From 07ade2e2384a6bd58b8de6b6e59647fda0d81b97 Mon Sep 17 00:00:00 2001 From: Jason Dai Date: Thu, 23 Jul 2026 17:38:07 -0700 Subject: [PATCH] feat: GenAI Client(evals) - add create EvaluationExperiment PiperOrigin-RevId: 953047538 --- agentplatform/_genai/evals.py | 221 +++++++++++++++++++ agentplatform/_genai/types/__init__.py | 16 +- agentplatform/_genai/types/common.py | 214 +++++++++++------- tests/unit/agentplatform/genai/test_evals.py | 53 +++++ vertexai/_genai/evals.py | 209 ++++++++++++++++++ vertexai/_genai/types/__init__.py | 8 + vertexai/_genai/types/common.py | 58 +++++ 7 files changed, 697 insertions(+), 82 deletions(-) diff --git a/agentplatform/_genai/evals.py b/agentplatform/_genai/evals.py index 1e1fd5c617..55d811edde 100644 --- a/agentplatform/_genai/evals.py +++ b/agentplatform/_genai/evals.py @@ -43,6 +43,29 @@ logger = logging.getLogger("agentplatform_genai.evals") +def _CreateEvaluationExperimentParameters_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["display_name"]) is not None: + setv(to_object, ["displayName"], getv(from_object, ["display_name"])) + + if getv(from_object, ["labels"]) is not None: + setv(to_object, ["labels"], getv(from_object, ["labels"])) + + if getv(from_object, ["merge_strategy"]) is not None: + setv(to_object, ["mergeStrategy"], getv(from_object, ["merge_strategy"])) + + if getv(from_object, ["metadata"]) is not None: + setv(to_object, ["metadata"], getv(from_object, ["metadata"])) + + if getv(from_object, ["config"]) is not None: + setv(to_object, ["config"], getv(from_object, ["config"])) + + return to_object + + def _CreateEvaluationItemParameters_to_vertex( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -1070,6 +1093,104 @@ def _UnifiedMetric_to_vertex( class Evals(_api_module.BaseModule): + def create_evaluation_experiment( + self, + *, + display_name: Optional[str] = None, + labels: Optional[dict[str, str]] = None, + merge_strategy: Optional[types.EvaluationExperimentMergeStrategy] = None, + metadata: Optional[dict[str, Any]] = None, + config: Optional[types.CreateEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """ + Creates an EvaluationExperiment. + + Args: + display_name: The display name of the evaluation experiment. + labels: Labels for the evaluation experiment. + merge_strategy: Merge strategy for the evaluation experiment. + metadata: Metadata about the evaluation experiment, can be used by the + caller to store additional tracking information about the experiment. + config: Optional configuration for the create operation. + + Returns: + The created evaluation experiment. + + .. code-block:: python + + eval_experiment = client.evals.create_evaluation_experiment( + display_name="my-experiment" + ) + + """ + + parameter_model = types._CreateEvaluationExperimentParameters( + display_name=display_name, + labels=labels, + merge_strategy=merge_strategy, + metadata=metadata, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _CreateEvaluationExperimentParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments".format_map(request_url_dict) + else: + path = "evaluationExperiments" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = self._api_client.request("post", path, request_dict, http_options) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.EvaluationExperiment._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + def _create_evaluation_item( self, *, @@ -3403,6 +3524,106 @@ def delete_evaluation_metric( class AsyncEvals(_api_module.BaseModule): + async def create_evaluation_experiment( + self, + *, + display_name: Optional[str] = None, + labels: Optional[dict[str, str]] = None, + merge_strategy: Optional[types.EvaluationExperimentMergeStrategy] = None, + metadata: Optional[dict[str, Any]] = None, + config: Optional[types.CreateEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """ + Creates an EvaluationExperiment. + + Args: + display_name: The display name of the evaluation experiment. + labels: Labels for the evaluation experiment. + merge_strategy: Merge strategy for the evaluation experiment. + metadata: Metadata about the evaluation experiment, can be used by the + caller to store additional tracking information about the experiment. + config: Optional configuration for the create operation. + + Returns: + The created evaluation experiment. + + .. code-block:: python + + eval_experiment = client.evals.create_evaluation_experiment( + display_name="my-experiment" + ) + + """ + + parameter_model = types._CreateEvaluationExperimentParameters( + display_name=display_name, + labels=labels, + merge_strategy=merge_strategy, + metadata=metadata, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _CreateEvaluationExperimentParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments".format_map(request_url_dict) + else: + path = "evaluationExperiments" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = await self._api_client.async_request( + "post", path, request_dict, http_options + ) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.EvaluationExperiment._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + async def _create_evaluation_item( self, *, diff --git a/agentplatform/_genai/types/__init__.py b/agentplatform/_genai/types/__init__.py index a8db3865fc..e7612f28d1 100644 --- a/agentplatform/_genai/types/__init__.py +++ b/agentplatform/_genai/types/__init__.py @@ -36,6 +36,7 @@ from .common import _CreateAgentEngineTaskRequestParameters from .common import _CreateDatasetParameters from .common import _CreateDatasetVersionParameters +from .common import _CreateEvaluationExperimentParameters from .common import _CreateEvaluationItemParameters from .common import _CreateEvaluationMetricParameters from .common import _CreateEvaluationRunParameters @@ -342,6 +343,9 @@ from .common import CreateDatasetVersionConfig from .common import CreateDatasetVersionConfigDict from .common import CreateDatasetVersionConfigOrDict +from .common import CreateEvaluationExperimentConfig +from .common import CreateEvaluationExperimentConfigDict +from .common import CreateEvaluationExperimentConfigOrDict from .common import CreateEvaluationItemConfig from .common import CreateEvaluationItemConfigDict from .common import CreateEvaluationItemConfigOrDict @@ -2013,6 +2017,12 @@ "ListAgentEngineTaskEventsResponse", "ListAgentEngineTaskEventsResponseDict", "ListAgentEngineTaskEventsResponseOrDict", + "CreateEvaluationExperimentConfig", + "CreateEvaluationExperimentConfigDict", + "CreateEvaluationExperimentConfigOrDict", + "EvaluationExperiment", + "EvaluationExperimentDict", + "EvaluationExperimentOrDict", "CreateEvaluationItemConfig", "CreateEvaluationItemConfigDict", "CreateEvaluationItemConfigOrDict", @@ -2337,9 +2347,6 @@ "GetEvaluationExperimentConfig", "GetEvaluationExperimentConfigDict", "GetEvaluationExperimentConfigOrDict", - "EvaluationExperiment", - "EvaluationExperimentDict", - "EvaluationExperimentOrDict", "GetEvaluationMetricConfig", "GetEvaluationMetricConfigDict", "GetEvaluationMetricConfigOrDict", @@ -3669,10 +3676,10 @@ "VersionState", "QuotaState", "FeedbackType", + "EvaluationExperimentMergeStrategy", "EvaluationItemType", "SamplingMethod", "EvaluationRunState", - "EvaluationExperimentMergeStrategy", "OptimizeTarget", "MemoryMetadataMergeStrategy", "GenerateMemoriesResponseGeneratedMemoryAction", @@ -3708,6 +3715,7 @@ "_CreateAgentEngineTaskRequestParameters", "_AppendAgentEngineTaskEventRequestParameters", "_ListAgentEngineTaskEventsRequestParameters", + "_CreateEvaluationExperimentParameters", "_CreateEvaluationItemParameters", "_CreateEvaluationMetricParameters", "_CreateEvaluationRunParameters", diff --git a/agentplatform/_genai/types/common.py b/agentplatform/_genai/types/common.py index 9188c69be1..cd6fb70b6c 100644 --- a/agentplatform/_genai/types/common.py +++ b/agentplatform/_genai/types/common.py @@ -498,6 +498,17 @@ class FeedbackType(_common.CaseInSensitiveEnum): """Indicates a thumbs down feedback (e.g., a "thumbs down").""" +class EvaluationExperimentMergeStrategy(_common.CaseInSensitiveEnum): + """Merge strategy for the evaluation experiment.""" + + MERGE_STRATEGY_UNSPECIFIED = "MERGE_STRATEGY_UNSPECIFIED" + """Unspecified merge strategy.""" + SEQUENTIAL_HISTORY = "SEQUENTIAL_HISTORY" + """Default. Runs are treated as an independent, sequential history.""" + SHARED_RESULT_SET = "SHARED_RESULT_SET" + """Runs are parallel iterations contributing to a shared result set.""" + + class EvaluationItemType(_common.CaseInSensitiveEnum): """The type of the EvaluationItem.""" @@ -539,17 +550,6 @@ class EvaluationRunState(_common.CaseInSensitiveEnum): """Evaluation run is performing rubric generation.""" -class EvaluationExperimentMergeStrategy(_common.CaseInSensitiveEnum): - """Merge strategy for the evaluation experiment.""" - - MERGE_STRATEGY_UNSPECIFIED = "MERGE_STRATEGY_UNSPECIFIED" - """Unspecified merge strategy.""" - SEQUENTIAL_HISTORY = "SEQUENTIAL_HISTORY" - """Default. Runs are treated as an independent, sequential history.""" - SHARED_RESULT_SET = "SHARED_RESULT_SET" - """Runs are parallel iterations contributing to a shared result set.""" - - class OptimizeTarget(_common.CaseInSensitiveEnum): """Specifies the method for calling the optimize_prompt.""" @@ -1498,6 +1498,131 @@ class ListAgentEngineTaskEventsResponseDict(TypedDict, total=False): ] +class CreateEvaluationExperimentConfig(_common.BaseModel): + """Config to create an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptions] = Field( + default=None, description="""Used to override HTTP request options.""" + ) + + +class CreateEvaluationExperimentConfigDict(TypedDict, total=False): + """Config to create an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptions] + """Used to override HTTP request options.""" + + +CreateEvaluationExperimentConfigOrDict = Union[ + CreateEvaluationExperimentConfig, CreateEvaluationExperimentConfigDict +] + + +class _CreateEvaluationExperimentParameters(_common.BaseModel): + """Parameters for creating an evaluation experiment.""" + + display_name: Optional[str] = Field(default=None, description="""""") + labels: Optional[dict[str, str]] = Field(default=None, description="""""") + merge_strategy: Optional[EvaluationExperimentMergeStrategy] = Field( + default=None, description="""""" + ) + metadata: Optional[dict[str, Any]] = Field(default=None, description="""""") + config: Optional[CreateEvaluationExperimentConfig] = Field( + default=None, description="""""" + ) + + +class _CreateEvaluationExperimentParametersDict(TypedDict, total=False): + """Parameters for creating an evaluation experiment.""" + + display_name: Optional[str] + """""" + + labels: Optional[dict[str, str]] + """""" + + merge_strategy: Optional[EvaluationExperimentMergeStrategy] + """""" + + metadata: Optional[dict[str, Any]] + """""" + + config: Optional[CreateEvaluationExperimentConfigDict] + """""" + + +_CreateEvaluationExperimentParametersOrDict = Union[ + _CreateEvaluationExperimentParameters, _CreateEvaluationExperimentParametersDict +] + + +class EvaluationExperiment(_common.BaseModel): + """Represents an experiment for iterating on and visualizing evaluation runs.""" + + name: Optional[str] = Field( + default=None, + description="""The resource name of the EvaluationExperiment. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`.""", + ) + display_name: Optional[str] = Field( + default=None, description="""The display name of the evaluation experiment.""" + ) + evaluation_runs: Optional[list[str]] = Field( + default=None, + description="""The EvaluationRuns that are part of this experiment.""", + ) + labels: Optional[dict[str, str]] = Field( + default=None, description="""Labels for the evaluation experiment.""" + ) + merge_strategy: Optional[EvaluationExperimentMergeStrategy] = Field( + default=None, description="""Merge strategy for the evaluation experiment.""" + ) + metadata: Optional[dict[str, Any]] = Field( + default=None, + description="""Metadata about the evaluation experiment, can be used by the caller + to store additional tracking information about the experiment.""", + ) + create_time: Optional[datetime.datetime] = Field( + default=None, description="""Timestamp when this experiment was created.""" + ) + update_time: Optional[datetime.datetime] = Field( + default=None, description="""Timestamp when this experiment was last updated.""" + ) + + +class EvaluationExperimentDict(TypedDict, total=False): + """Represents an experiment for iterating on and visualizing evaluation runs.""" + + name: Optional[str] + """The resource name of the EvaluationExperiment. Format: + `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`.""" + + display_name: Optional[str] + """The display name of the evaluation experiment.""" + + evaluation_runs: Optional[list[str]] + """The EvaluationRuns that are part of this experiment.""" + + labels: Optional[dict[str, str]] + """Labels for the evaluation experiment.""" + + merge_strategy: Optional[EvaluationExperimentMergeStrategy] + """Merge strategy for the evaluation experiment.""" + + metadata: Optional[dict[str, Any]] + """Metadata about the evaluation experiment, can be used by the caller + to store additional tracking information about the experiment.""" + + create_time: Optional[datetime.datetime] + """Timestamp when this experiment was created.""" + + update_time: Optional[datetime.datetime] + """Timestamp when this experiment was last updated.""" + + +EvaluationExperimentOrDict = Union[EvaluationExperiment, EvaluationExperimentDict] + + class CreateEvaluationItemConfig(_common.BaseModel): """Config to create an evaluation item.""" @@ -5849,73 +5974,6 @@ class _GetEvaluationExperimentParametersDict(TypedDict, total=False): ] -class EvaluationExperiment(_common.BaseModel): - """Represents an experiment for iterating on and visualizing evaluation runs.""" - - name: Optional[str] = Field( - default=None, - description="""The resource name of the EvaluationExperiment. Format: - `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`.""", - ) - display_name: Optional[str] = Field( - default=None, description="""The display name of the evaluation experiment.""" - ) - evaluation_runs: Optional[list[str]] = Field( - default=None, - description="""The EvaluationRuns that are part of this experiment.""", - ) - labels: Optional[dict[str, str]] = Field( - default=None, description="""Labels for the evaluation experiment.""" - ) - merge_strategy: Optional[EvaluationExperimentMergeStrategy] = Field( - default=None, description="""Merge strategy for the evaluation experiment.""" - ) - metadata: Optional[dict[str, Any]] = Field( - default=None, - description="""Metadata about the evaluation experiment, can be used by the caller - to store additional tracking information about the experiment.""", - ) - create_time: Optional[datetime.datetime] = Field( - default=None, description="""Timestamp when this experiment was created.""" - ) - update_time: Optional[datetime.datetime] = Field( - default=None, description="""Timestamp when this experiment was last updated.""" - ) - - -class EvaluationExperimentDict(TypedDict, total=False): - """Represents an experiment for iterating on and visualizing evaluation runs.""" - - name: Optional[str] - """The resource name of the EvaluationExperiment. Format: - `projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`.""" - - display_name: Optional[str] - """The display name of the evaluation experiment.""" - - evaluation_runs: Optional[list[str]] - """The EvaluationRuns that are part of this experiment.""" - - labels: Optional[dict[str, str]] - """Labels for the evaluation experiment.""" - - merge_strategy: Optional[EvaluationExperimentMergeStrategy] - """Merge strategy for the evaluation experiment.""" - - metadata: Optional[dict[str, Any]] - """Metadata about the evaluation experiment, can be used by the caller - to store additional tracking information about the experiment.""" - - create_time: Optional[datetime.datetime] - """Timestamp when this experiment was created.""" - - update_time: Optional[datetime.datetime] - """Timestamp when this experiment was last updated.""" - - -EvaluationExperimentOrDict = Union[EvaluationExperiment, EvaluationExperimentDict] - - class GetEvaluationMetricConfig(_common.BaseModel): """Config for getting an evaluation metric.""" diff --git a/tests/unit/agentplatform/genai/test_evals.py b/tests/unit/agentplatform/genai/test_evals.py index 0e3e69f9d3..6c952203f3 100644 --- a/tests/unit/agentplatform/genai/test_evals.py +++ b/tests/unit/agentplatform/genai/test_evals.py @@ -11833,3 +11833,56 @@ def test_list_evaluation_experiments_passes_filter_and_order_by(self): path = self.mock_api_client.request.call_args[0][1] assert path.startswith("evaluationExperiments?") assert "orderBy=create_time+desc" in path + + +class TestCreateEvaluationExperiment: + + def setup_method(self, method): + self.mock_api_client = mock.MagicMock() + self.mock_api_client.vertexai = True + self.mock_response = mock.MagicMock() + self.mock_response.body = json.dumps( + { + "name": "projects/123/locations/us-central1/evaluationExperiments/456", + "displayName": "my_experiment", + } + ) + self.mock_api_client.request.return_value = self.mock_response + + def test_create_evaluation_experiment_returns_experiment(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + experiment = evals_module.create_evaluation_experiment( + display_name="my_experiment" + ) + + assert isinstance(experiment, agentplatform_genai_types.EvaluationExperiment) + assert experiment.display_name == "my_experiment" + + def test_create_evaluation_experiment_posts_to_experiments(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + evals_module.create_evaluation_experiment(display_name="my_experiment") + + self.mock_api_client.request.assert_called_once() + call_args = self.mock_api_client.request.call_args + assert call_args[0][0] == "post" + assert call_args[0][1] == "evaluationExperiments" + request_body = call_args[0][2] + assert request_body.get("displayName") == "my_experiment" + + def test_create_evaluation_experiment_passes_all_params(self): + evals_module = evals.Evals(api_client_=self.mock_api_client) + + evals_module.create_evaluation_experiment( + display_name="my_experiment", + merge_strategy=agentplatform_genai_types.EvaluationExperimentMergeStrategy.SHARED_RESULT_SET, + labels={"team": "agents"}, + metadata={"owner": "test"}, + ) + + request_body = self.mock_api_client.request.call_args[0][2] + assert request_body.get("displayName") == "my_experiment" + assert request_body.get("mergeStrategy") == "SHARED_RESULT_SET" + assert request_body.get("labels") == {"team": "agents"} + assert request_body.get("metadata") == {"owner": "test"} diff --git a/vertexai/_genai/evals.py b/vertexai/_genai/evals.py index 5078d0265a..9c78c5f04b 100644 --- a/vertexai/_genai/evals.py +++ b/vertexai/_genai/evals.py @@ -140,6 +140,29 @@ def _CreateEvaluationRunParameters_to_vertex( return to_object +def _CreateEvaluationExperimentParameters_to_vertex( + from_object: Union[dict[str, Any], object], + parent_object: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + to_object: dict[str, Any] = {} + if getv(from_object, ["display_name"]) is not None: + setv(to_object, ["displayName"], getv(from_object, ["display_name"])) + + if getv(from_object, ["labels"]) is not None: + setv(to_object, ["labels"], getv(from_object, ["labels"])) + + if getv(from_object, ["merge_strategy"]) is not None: + setv(to_object, ["mergeStrategy"], getv(from_object, ["merge_strategy"])) + + if getv(from_object, ["metadata"]) is not None: + setv(to_object, ["metadata"], getv(from_object, ["metadata"])) + + if getv(from_object, ["config"]) is not None: + setv(to_object, ["config"], getv(from_object, ["config"])) + + return to_object + + def _CreateEvaluationSetParameters_to_vertex( from_object: Union[dict[str, Any], object], parent_object: Optional[dict[str, Any]] = None, @@ -3260,6 +3283,98 @@ def list_evaluation_experiments( self._api_client._verify_response(return_value) return return_value + def create_evaluation_experiment( + self, + *, + display_name: Optional[str] = None, + labels: Optional[dict[str, str]] = None, + merge_strategy: Optional[types.EvaluationExperimentMergeStrategy] = None, + metadata: Optional[dict[str, Any]] = None, + config: Optional[types.CreateEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """ + Creates an EvaluationExperiment. + + Args: + display_name: The display name of the evaluation experiment. + labels: Labels for the evaluation experiment. + merge_strategy: Merge strategy for the evaluation experiment. + metadata: Metadata about the evaluation experiment, can be used by the + caller to store additional tracking information about the experiment. + config: Optional configuration for the create operation. + + Returns: + The created evaluation experiment. + + """ + + parameter_model = types._CreateEvaluationExperimentParameters( + display_name=display_name, + labels=labels, + merge_strategy=merge_strategy, + metadata=metadata, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _CreateEvaluationExperimentParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments".format_map(request_url_dict) + else: + path = "evaluationExperiments" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = self._api_client.request("post", path, request_dict, http_options) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.EvaluationExperiment._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + def get_evaluation_metric( self, *, @@ -5188,6 +5303,100 @@ async def list_evaluation_experiments( self._api_client._verify_response(return_value) return return_value + async def create_evaluation_experiment( + self, + *, + display_name: Optional[str] = None, + labels: Optional[dict[str, str]] = None, + merge_strategy: Optional[types.EvaluationExperimentMergeStrategy] = None, + metadata: Optional[dict[str, Any]] = None, + config: Optional[types.CreateEvaluationExperimentConfigOrDict] = None, + ) -> types.EvaluationExperiment: + """ + Creates an EvaluationExperiment. + + Args: + display_name: The display name of the evaluation experiment. + labels: Labels for the evaluation experiment. + merge_strategy: Merge strategy for the evaluation experiment. + metadata: Metadata about the evaluation experiment, can be used by the + caller to store additional tracking information about the experiment. + config: Optional configuration for the create operation. + + Returns: + The created evaluation experiment. + + """ + + parameter_model = types._CreateEvaluationExperimentParameters( + display_name=display_name, + labels=labels, + merge_strategy=merge_strategy, + metadata=metadata, + config=config, + ) + + request_url_dict: Optional[dict[str, str]] + if not self._api_client.vertexai: + raise ValueError( + "This method is only supported in Gemini Enterprise Agent Platform mode, not in Gemini Developer API mode." + ) + else: + request_dict = _CreateEvaluationExperimentParameters_to_vertex( + parameter_model + ) + request_url_dict = request_dict.get("_url") + if request_url_dict: + path = "evaluationExperiments".format_map(request_url_dict) + else: + path = "evaluationExperiments" + + query_params = request_dict.get("_query") + if query_params: + path = f"{path}?{urlencode(query_params)}" + # TODO: remove the hack that pops config. + request_dict.pop("config", None) + + http_options: Optional[types.HttpOptions] = None + if ( + parameter_model.config is not None + and parameter_model.config.http_options is not None + ): + http_options = parameter_model.config.http_options + + request_dict = _common.convert_to_dict(request_dict) + request_dict = _common.encode_unserializable_types(request_dict) + + response = await self._api_client.async_request( + "post", path, request_dict, http_options + ) + + response_dict = {} if not response.body else json.loads(response.body) + + return_value = types.EvaluationExperiment._from_response( + response=response_dict, + kwargs=( + { + "config": { + "response_schema": getattr( + parameter_model.config, "response_schema", None + ), + "response_json_schema": getattr( + parameter_model.config, "response_json_schema", None + ), + "include_all_fields": getattr( + parameter_model.config, "include_all_fields", None + ), + } + } + if getattr(parameter_model, "config", None) + else {} + ), + ) + + self._api_client._verify_response(return_value) + return return_value + async def get_evaluation_metric( self, *, diff --git a/vertexai/_genai/types/__init__.py b/vertexai/_genai/types/__init__.py index 040a6323d3..f692f2351d 100644 --- a/vertexai/_genai/types/__init__.py +++ b/vertexai/_genai/types/__init__.py @@ -297,6 +297,10 @@ from .common import CreateEvaluationMetricConfig from .common import CreateEvaluationMetricConfigDict from .common import CreateEvaluationMetricConfigOrDict +from .common import CreateEvaluationExperimentConfig +from .common import CreateEvaluationExperimentConfigDict +from .common import CreateEvaluationExperimentConfigOrDict +from .common import _CreateEvaluationExperimentParameters from .common import CreateEvaluationRunConfig from .common import CreateEvaluationRunConfigDict from .common import CreateEvaluationRunConfigOrDict @@ -1617,6 +1621,9 @@ "AnalysisConfig", "AnalysisConfigDict", "AnalysisConfigOrDict", + "CreateEvaluationExperimentConfig", + "CreateEvaluationExperimentConfigDict", + "CreateEvaluationExperimentConfigOrDict", "CreateEvaluationRunConfig", "CreateEvaluationRunConfigDict", "CreateEvaluationRunConfigOrDict", @@ -2798,6 +2805,7 @@ "_CreateAgentEngineTaskRequestParameters", "_AppendAgentEngineTaskEventRequestParameters", "_ListAgentEngineTaskEventsRequestParameters", + "_CreateEvaluationExperimentParameters", "_CreateEvaluationItemParameters", "_CreateEvaluationMetricParameters", "_CreateEvaluationRunParameters", diff --git a/vertexai/_genai/types/common.py b/vertexai/_genai/types/common.py index 0cc660614a..3791b6ad13 100644 --- a/vertexai/_genai/types/common.py +++ b/vertexai/_genai/types/common.py @@ -1383,6 +1383,64 @@ class ListAgentEngineTaskEventsResponseDict(TypedDict, total=False): ] +class CreateEvaluationExperimentConfig(_common.BaseModel): + """Config to create an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptions] = Field( + default=None, description="""Used to override HTTP request options.""" + ) + + +class CreateEvaluationExperimentConfigDict(TypedDict, total=False): + """Config to create an evaluation experiment.""" + + http_options: Optional[genai_types.HttpOptionsDict] + """Used to override HTTP request options.""" + + +CreateEvaluationExperimentConfigOrDict = Union[ + CreateEvaluationExperimentConfig, CreateEvaluationExperimentConfigDict +] + + +class _CreateEvaluationExperimentParameters(_common.BaseModel): + """Parameters for creating an evaluation experiment.""" + + display_name: Optional[str] = Field(default=None, description="""""") + labels: Optional[dict[str, str]] = Field(default=None, description="""""") + merge_strategy: Optional[EvaluationExperimentMergeStrategy] = Field( + default=None, description="""""" + ) + metadata: Optional[dict[str, Any]] = Field(default=None, description="""""") + config: Optional[CreateEvaluationExperimentConfig] = Field( + default=None, description="""""" + ) + + +class _CreateEvaluationExperimentParametersDict(TypedDict, total=False): + """Parameters for creating an evaluation experiment.""" + + display_name: Optional[str] + """""" + + labels: Optional[dict[str, str]] + """""" + + merge_strategy: Optional[EvaluationExperimentMergeStrategy] + """""" + + metadata: Optional[dict[str, Any]] + """""" + + config: Optional[CreateEvaluationExperimentConfigDict] + """""" + + +_CreateEvaluationExperimentParametersOrDict = Union[ + _CreateEvaluationExperimentParameters, _CreateEvaluationExperimentParametersDict +] + + class CreateEvaluationItemConfig(_common.BaseModel): """Config to create an evaluation item."""