From 7f71f8ba9dd18c1da2d4fc92573b5b508fb6eb6c Mon Sep 17 00:00:00 2001 From: James Chainey Date: Wed, 4 Feb 2026 17:24:45 -0800 Subject: [PATCH 1/3] removed custom scorers --- .../resources/scenarios/scorers.py | 122 +----------------- src/runloop_api_client/sdk/_types.py | 6 +- .../sdk/async_scenario_builder.py | 30 ----- src/runloop_api_client/sdk/async_scorer.py | 15 +-- .../sdk/scenario_builder.py | 30 ----- src/runloop_api_client/sdk/scorer.py | 15 +-- .../types/scenarios/__init__.py | 2 - .../types/scenarios/scorer_validate_params.py | 17 --- .../scenarios/scorer_validate_response.py | 23 ---- .../types/scoring_function.py | 14 -- .../types/scoring_function_param.py | 14 -- tests/sdk/test_async_scenario_builder.py | 10 +- tests/sdk/test_async_scorer.py | 17 --- tests/sdk/test_scenario_builder.py | 10 +- tests/sdk/test_scorer.py | 19 --- 15 files changed, 8 insertions(+), 336 deletions(-) delete mode 100644 src/runloop_api_client/types/scenarios/scorer_validate_params.py delete mode 100644 src/runloop_api_client/types/scenarios/scorer_validate_response.py diff --git a/src/runloop_api_client/resources/scenarios/scorers.py b/src/runloop_api_client/resources/scenarios/scorers.py index 9e5d5e198..cdb011dc7 100644 --- a/src/runloop_api_client/resources/scenarios/scorers.py +++ b/src/runloop_api_client/resources/scenarios/scorers.py @@ -16,13 +16,11 @@ ) from ...pagination import SyncScenarioScorersCursorIDPage, AsyncScenarioScorersCursorIDPage from ..._base_client import AsyncPaginator, make_request_options -from ...types.scenarios import scorer_list_params, scorer_create_params, scorer_update_params, scorer_validate_params -from ...types.scenario_environment_param import ScenarioEnvironmentParam +from ...types.scenarios import scorer_list_params, scorer_create_params, scorer_update_params from ...types.scenarios.scorer_list_response import ScorerListResponse from ...types.scenarios.scorer_create_response import ScorerCreateResponse from ...types.scenarios.scorer_update_response import ScorerUpdateResponse from ...types.scenarios.scorer_retrieve_response import ScorerRetrieveResponse -from ...types.scenarios.scorer_validate_response import ScorerValidateResponse __all__ = ["ScorersResource", "AsyncScorersResource"] @@ -232,59 +230,6 @@ def list( model=ScorerListResponse, ) - def validate( - self, - id: str, - *, - scoring_context: object, - environment_parameters: ScenarioEnvironmentParam | Omit = omit, - # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. - # The extra values given here take precedence over values defined on the client or passed to this method. - extra_headers: Headers | None = None, - extra_query: Query | None = None, - extra_body: Body | None = None, - timeout: float | httpx.Timeout | None | NotGiven = not_given, - idempotency_key: str | None = None, - ) -> ScorerValidateResponse: - """ - Validate a scenario scorer. - - Args: - scoring_context: Json context that gets passed to the custom scorer - - environment_parameters: The Environment in which the Scenario will run. - - extra_headers: Send extra headers - - extra_query: Add additional query parameters to the request - - extra_body: Add additional JSON properties to the request - - timeout: Override the client-level default timeout for this request, in seconds - - idempotency_key: Specify a custom idempotency key for this request - """ - if not id: - raise ValueError(f"Expected a non-empty value for `id` but received {id!r}") - return self._post( - f"/v1/scenarios/scorers/{id}/validate", - body=maybe_transform( - { - "scoring_context": scoring_context, - "environment_parameters": environment_parameters, - }, - scorer_validate_params.ScorerValidateParams, - ), - options=make_request_options( - extra_headers=extra_headers, - extra_query=extra_query, - extra_body=extra_body, - timeout=timeout, - idempotency_key=idempotency_key, - ), - cast_to=ScorerValidateResponse, - ) - class AsyncScorersResource(AsyncAPIResource): @cached_property @@ -491,59 +436,6 @@ def list( model=ScorerListResponse, ) - async def validate( - self, - id: str, - *, - scoring_context: object, - environment_parameters: ScenarioEnvironmentParam | Omit = omit, - # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. - # The extra values given here take precedence over values defined on the client or passed to this method. - extra_headers: Headers | None = None, - extra_query: Query | None = None, - extra_body: Body | None = None, - timeout: float | httpx.Timeout | None | NotGiven = not_given, - idempotency_key: str | None = None, - ) -> ScorerValidateResponse: - """ - Validate a scenario scorer. - - Args: - scoring_context: Json context that gets passed to the custom scorer - - environment_parameters: The Environment in which the Scenario will run. - - extra_headers: Send extra headers - - extra_query: Add additional query parameters to the request - - extra_body: Add additional JSON properties to the request - - timeout: Override the client-level default timeout for this request, in seconds - - idempotency_key: Specify a custom idempotency key for this request - """ - if not id: - raise ValueError(f"Expected a non-empty value for `id` but received {id!r}") - return await self._post( - f"/v1/scenarios/scorers/{id}/validate", - body=await async_maybe_transform( - { - "scoring_context": scoring_context, - "environment_parameters": environment_parameters, - }, - scorer_validate_params.ScorerValidateParams, - ), - options=make_request_options( - extra_headers=extra_headers, - extra_query=extra_query, - extra_body=extra_body, - timeout=timeout, - idempotency_key=idempotency_key, - ), - cast_to=ScorerValidateResponse, - ) - class ScorersResourceWithRawResponse: def __init__(self, scorers: ScorersResource) -> None: @@ -561,9 +453,6 @@ def __init__(self, scorers: ScorersResource) -> None: self.list = to_raw_response_wrapper( scorers.list, ) - self.validate = to_raw_response_wrapper( - scorers.validate, - ) class AsyncScorersResourceWithRawResponse: @@ -582,9 +471,6 @@ def __init__(self, scorers: AsyncScorersResource) -> None: self.list = async_to_raw_response_wrapper( scorers.list, ) - self.validate = async_to_raw_response_wrapper( - scorers.validate, - ) class ScorersResourceWithStreamingResponse: @@ -603,9 +489,6 @@ def __init__(self, scorers: ScorersResource) -> None: self.list = to_streamed_response_wrapper( scorers.list, ) - self.validate = to_streamed_response_wrapper( - scorers.validate, - ) class AsyncScorersResourceWithStreamingResponse: @@ -624,6 +507,3 @@ def __init__(self, scorers: AsyncScorersResource) -> None: self.list = async_to_streamed_response_wrapper( scorers.list, ) - self.validate = async_to_streamed_response_wrapper( - scorers.validate, - ) diff --git a/src/runloop_api_client/sdk/_types.py b/src/runloop_api_client/sdk/_types.py index 9eb0526bc..9e7f8780e 100644 --- a/src/runloop_api_client/sdk/_types.py +++ b/src/runloop_api_client/sdk/_types.py @@ -37,7 +37,7 @@ from .._types import Body, Query, Headers, Timeout, NotGiven from ..lib.polling import PollingConfig from ..types.devboxes import DiskSnapshotListParams, DiskSnapshotUpdateParams -from ..types.scenarios import ScorerListParams, ScorerCreateParams, ScorerUpdateParams, ScorerValidateParams +from ..types.scenarios import ScorerListParams, ScorerCreateParams, ScorerUpdateParams from ..types.devbox_create_params import DevboxBaseCreateParams from ..types.scenario_start_run_params import ScenarioStartRunBaseParams from ..types.benchmark_start_run_params import BenchmarkSelfStartRunParams @@ -181,10 +181,6 @@ class SDKScorerUpdateParams(ScorerUpdateParams, LongRequestOptions): pass -class SDKScorerValidateParams(ScorerValidateParams, LongRequestOptions): - pass - - class SDKAgentCreateParams(AgentCreateParams, LongRequestOptions): pass diff --git a/src/runloop_api_client/sdk/async_scenario_builder.py b/src/runloop_api_client/sdk/async_scenario_builder.py index 37a3aa4b5..18d02ae14 100644 --- a/src/runloop_api_client/sdk/async_scenario_builder.py +++ b/src/runloop_api_client/sdk/async_scenario_builder.py @@ -14,7 +14,6 @@ from ..types.scoring_function_param import ( Scorer, ScoringFunctionParam, - ScorerCustomScoringFunction, ScorerAstGrepScoringFunction, ScorerCommandScoringFunction, ScorerTestBasedScoringFunction, @@ -304,35 +303,6 @@ def add_ast_grep_scorer( scorer["lang"] = lang return self._add_scorer(name, weight, scorer) - def add_custom_scorer( - self, - name: str, - *, - custom_scorer_type: str, - weight: float = 1.0, - scorer_params: Optional[object] = None, - ) -> Self: - """Add a custom scorer registered with Runloop. - - :param name: Name of the scoring function - :type name: str - :param custom_scorer_type: Type identifier registered with Runloop - :type custom_scorer_type: str - :param weight: Weight for this scorer (normalized automatically) - :type weight: float - :param scorer_params: Additional JSON parameters for the scorer - :type scorer_params: Optional[object] - :return: Self for method chaining - :rtype: Self - """ - scorer: ScorerCustomScoringFunction = { - "type": "custom_scorer", - "custom_scorer_type": custom_scorer_type, - } - if scorer_params: - scorer["scorer_params"] = scorer_params - return self._add_scorer(name, weight, scorer) - def with_metadata(self, metadata: Dict[str, str]) -> Self: """Set metadata for the scenario. diff --git a/src/runloop_api_client/sdk/async_scorer.py b/src/runloop_api_client/sdk/async_scorer.py index 91ced0c38..50af08eaa 100644 --- a/src/runloop_api_client/sdk/async_scorer.py +++ b/src/runloop_api_client/sdk/async_scorer.py @@ -7,14 +7,13 @@ from ._types import ( BaseRequestOptions, SDKScorerUpdateParams, - SDKScorerValidateParams, ) from .._client import AsyncRunloop -from ..types.scenarios import ScorerUpdateResponse, ScorerRetrieveResponse, ScorerValidateResponse +from ..types.scenarios import ScorerUpdateResponse, ScorerRetrieveResponse class AsyncScorer: - """A custom scorer for evaluating scenario outputs (async). + """A scorer for evaluating scenario outputs (async). Scorers define bash scripts that produce a score in the range [0.0, 1.0] for scenario runs. Obtain instances via ``runloop.scorer.create()`` or ``runloop.scorer.from_id()``. @@ -22,7 +21,6 @@ class AsyncScorer: Example: >>> runloop = AsyncRunloopSDK() >>> scorer = await runloop.scorer.create(type="my_scorer", bash_script="echo 'score=1.0'") - >>> await scorer.validate(scoring_context={"output": "test"}) """ def __init__(self, client: AsyncRunloop, scorer_id: str) -> None: @@ -66,12 +64,3 @@ async def update(self, **params: Unpack[SDKScorerUpdateParams]) -> ScorerUpdateR :rtype: ScorerUpdateResponse """ return await self._client.scenarios.scorers.update(self._id, **params) - - async def validate(self, **params: Unpack[SDKScorerValidateParams]) -> ScorerValidateResponse: - """Run the scorer against the provided context and return the result. - - :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScorerValidateParams` for available parameters - :return: Validation result with score - :rtype: ScorerValidateResponse - """ - return await self._client.scenarios.scorers.validate(self._id, **params) diff --git a/src/runloop_api_client/sdk/scenario_builder.py b/src/runloop_api_client/sdk/scenario_builder.py index e2fc15de4..2d8b405ed 100644 --- a/src/runloop_api_client/sdk/scenario_builder.py +++ b/src/runloop_api_client/sdk/scenario_builder.py @@ -14,7 +14,6 @@ from ..types.scoring_function_param import ( Scorer, ScoringFunctionParam, - ScorerCustomScoringFunction, ScorerAstGrepScoringFunction, ScorerCommandScoringFunction, ScorerTestBasedScoringFunction, @@ -304,35 +303,6 @@ def add_ast_grep_scorer( scorer["lang"] = lang return self._add_scorer(name, weight, scorer) - def add_custom_scorer( - self, - name: str, - *, - custom_scorer_type: str, - weight: float = 1.0, - scorer_params: Optional[object] = None, - ) -> Self: - """Add a custom scorer registered with Runloop. - - :param name: Name of the scoring function - :type name: str - :param custom_scorer_type: Type identifier registered with Runloop - :type custom_scorer_type: str - :param weight: Weight for this scorer (normalized automatically) - :type weight: float - :param scorer_params: Additional JSON parameters for the scorer - :type scorer_params: Optional[object] - :return: Self for method chaining - :rtype: Self - """ - scorer: ScorerCustomScoringFunction = { - "type": "custom_scorer", - "custom_scorer_type": custom_scorer_type, - } - if scorer_params: - scorer["scorer_params"] = scorer_params - return self._add_scorer(name, weight, scorer) - def with_metadata(self, metadata: Dict[str, str]) -> Self: """Set metadata for the scenario. diff --git a/src/runloop_api_client/sdk/scorer.py b/src/runloop_api_client/sdk/scorer.py index 8df57ac05..37edbee40 100644 --- a/src/runloop_api_client/sdk/scorer.py +++ b/src/runloop_api_client/sdk/scorer.py @@ -7,14 +7,13 @@ from ._types import ( BaseRequestOptions, SDKScorerUpdateParams, - SDKScorerValidateParams, ) from .._client import Runloop -from ..types.scenarios import ScorerUpdateResponse, ScorerRetrieveResponse, ScorerValidateResponse +from ..types.scenarios import ScorerUpdateResponse, ScorerRetrieveResponse class Scorer: - """A custom scorer for evaluating scenario outputs. + """A scorer for evaluating scenario outputs. Scorers define bash scripts that produce a score in the range [0.0, 1.0] for scenario runs. Obtain instances via ``runloop.scorer.create()`` or ``runloop.scorer.from_id()``. @@ -22,7 +21,6 @@ class Scorer: Example: >>> runloop = RunloopSDK() >>> scorer = runloop.scorer.create(type="my_scorer", bash_script="echo 'score=1.0'") - >>> scorer.validate(scoring_context={"output": "test"}) """ def __init__(self, client: Runloop, scorer_id: str) -> None: @@ -66,12 +64,3 @@ def update(self, **params: Unpack[SDKScorerUpdateParams]) -> ScorerUpdateRespons :rtype: ScorerUpdateResponse """ return self._client.scenarios.scorers.update(self._id, **params) - - def validate(self, **params: Unpack[SDKScorerValidateParams]) -> ScorerValidateResponse: - """Run the scorer against the provided context and return the result. - - :param params: See :typeddict:`~runloop_api_client.sdk._types.SDKScorerValidateParams` for available parameters - :return: Validation result with score - :rtype: ScorerValidateResponse - """ - return self._client.scenarios.scorers.validate(self._id, **params) diff --git a/src/runloop_api_client/types/scenarios/__init__.py b/src/runloop_api_client/types/scenarios/__init__.py index d25c85c4e..97efebd9d 100644 --- a/src/runloop_api_client/types/scenarios/__init__.py +++ b/src/runloop_api_client/types/scenarios/__init__.py @@ -9,6 +9,4 @@ from .scorer_update_params import ScorerUpdateParams as ScorerUpdateParams from .scorer_create_response import ScorerCreateResponse as ScorerCreateResponse from .scorer_update_response import ScorerUpdateResponse as ScorerUpdateResponse -from .scorer_validate_params import ScorerValidateParams as ScorerValidateParams from .scorer_retrieve_response import ScorerRetrieveResponse as ScorerRetrieveResponse -from .scorer_validate_response import ScorerValidateResponse as ScorerValidateResponse diff --git a/src/runloop_api_client/types/scenarios/scorer_validate_params.py b/src/runloop_api_client/types/scenarios/scorer_validate_params.py deleted file mode 100644 index 41215ea2d..000000000 --- a/src/runloop_api_client/types/scenarios/scorer_validate_params.py +++ /dev/null @@ -1,17 +0,0 @@ -# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. - -from __future__ import annotations - -from typing_extensions import Required, TypedDict - -from ..scenario_environment_param import ScenarioEnvironmentParam - -__all__ = ["ScorerValidateParams"] - - -class ScorerValidateParams(TypedDict, total=False): - scoring_context: Required[object] - """Json context that gets passed to the custom scorer""" - - environment_parameters: ScenarioEnvironmentParam - """The Environment in which the Scenario will run.""" diff --git a/src/runloop_api_client/types/scenarios/scorer_validate_response.py b/src/runloop_api_client/types/scenarios/scorer_validate_response.py deleted file mode 100644 index 6c2755e6c..000000000 --- a/src/runloop_api_client/types/scenarios/scorer_validate_response.py +++ /dev/null @@ -1,23 +0,0 @@ -# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. - -from typing import Optional - -from ..._models import BaseModel -from ..scenario_environment import ScenarioEnvironment -from ..scoring_function_result_view import ScoringFunctionResultView - -__all__ = ["ScorerValidateResponse"] - - -class ScorerValidateResponse(BaseModel): - name: str - """Name of the custom scorer.""" - - scoring_context: object - """Json context that gets passed to the custom scorer""" - - scoring_result: ScoringFunctionResultView - """Result of the scoring function.""" - - environment_parameters: Optional[ScenarioEnvironment] = None - """The Environment in which the Scenario will run.""" diff --git a/src/runloop_api_client/types/scoring_function.py b/src/runloop_api_client/types/scoring_function.py index fe5d2a467..fd666925b 100644 --- a/src/runloop_api_client/types/scoring_function.py +++ b/src/runloop_api_client/types/scoring_function.py @@ -12,7 +12,6 @@ "ScorerAstGrepScoringFunction", "ScorerBashScriptScoringFunction", "ScorerCommandScoringFunction", - "ScorerCustomScoringFunction", "ScorerPythonScriptScoringFunction", "ScorerTestBasedScoringFunction", "ScorerTestBasedScoringFunctionTestFile", @@ -64,18 +63,6 @@ class ScorerCommandScoringFunction(BaseModel): """The command to execute.""" -class ScorerCustomScoringFunction(BaseModel): - """CustomScoringFunction is a custom, user defined scoring function.""" - - custom_scorer_type: str - """Type of the scoring function, previously registered with Runloop.""" - - type: Literal["custom_scorer"] - - scorer_params: Optional[object] = None - """Additional JSON structured context to pass to the scoring function.""" - - class ScorerPythonScriptScoringFunction(BaseModel): """ PythonScriptScoringFunction will run a python script in the context of your environment as a ScoringFunction. @@ -130,7 +117,6 @@ class ScorerTestBasedScoringFunction(BaseModel): ScorerAstGrepScoringFunction, ScorerBashScriptScoringFunction, ScorerCommandScoringFunction, - ScorerCustomScoringFunction, ScorerPythonScriptScoringFunction, ScorerTestBasedScoringFunction, ], diff --git a/src/runloop_api_client/types/scoring_function_param.py b/src/runloop_api_client/types/scoring_function_param.py index 033101d52..d485f8d4a 100644 --- a/src/runloop_api_client/types/scoring_function_param.py +++ b/src/runloop_api_client/types/scoring_function_param.py @@ -11,7 +11,6 @@ "ScorerAstGrepScoringFunction", "ScorerBashScriptScoringFunction", "ScorerCommandScoringFunction", - "ScorerCustomScoringFunction", "ScorerPythonScriptScoringFunction", "ScorerTestBasedScoringFunction", "ScorerTestBasedScoringFunctionTestFile", @@ -63,18 +62,6 @@ class ScorerCommandScoringFunction(TypedDict, total=False): """The command to execute.""" -class ScorerCustomScoringFunction(TypedDict, total=False): - """CustomScoringFunction is a custom, user defined scoring function.""" - - custom_scorer_type: Required[str] - """Type of the scoring function, previously registered with Runloop.""" - - type: Required[Literal["custom_scorer"]] - - scorer_params: Optional[object] - """Additional JSON structured context to pass to the scoring function.""" - - class ScorerPythonScriptScoringFunction(TypedDict, total=False): """ PythonScriptScoringFunction will run a python script in the context of your environment as a ScoringFunction. @@ -128,7 +115,6 @@ class ScorerTestBasedScoringFunction(TypedDict, total=False): ScorerAstGrepScoringFunction, ScorerBashScriptScoringFunction, ScorerCommandScoringFunction, - ScorerCustomScoringFunction, ScorerPythonScriptScoringFunction, ScorerTestBasedScoringFunction, ] diff --git a/tests/sdk/test_async_scenario_builder.py b/tests/sdk/test_async_scenario_builder.py index e20d99843..1050293c1 100644 --- a/tests/sdk/test_async_scenario_builder.py +++ b/tests/sdk/test_async_scenario_builder.py @@ -109,16 +109,8 @@ def test_scorers(self, mock_builder: AsyncScenarioBuilder) -> None: assert mock_builder._scorers[4]["scorer"].get("pattern") == "$A.foo()" assert mock_builder._scorers[4]["scorer"].get("lang") == "python" - # Custom scorer with optional params - mock_builder.add_custom_scorer( - "custom-scorer", custom_scorer_type="my_scorer", scorer_params={"threshold": 0.5} - ) - assert mock_builder._scorers[5]["scorer"]["type"] == "custom_scorer" - assert mock_builder._scorers[5]["scorer"].get("custom_scorer_type") == "my_scorer" - assert mock_builder._scorers[5]["scorer"].get("scorer_params") == {"threshold": 0.5} - # Verify multiple scorers accumulated - assert len(mock_builder._scorers) == 6 + assert len(mock_builder._scorers) == 5 def test_add_scorer_rejects_invalid_weight(self, mock_builder: AsyncScenarioBuilder) -> None: """Test that adding a scorer with zero or negative weight raises ValueError.""" diff --git a/tests/sdk/test_async_scorer.py b/tests/sdk/test_async_scorer.py index 253ae9585..637bba044 100644 --- a/tests/sdk/test_async_scorer.py +++ b/tests/sdk/test_async_scorer.py @@ -50,20 +50,3 @@ async def test_update(self, mock_async_client: AsyncMock) -> None: assert result == update_response mock_async_client.scenarios.scorers.update.assert_awaited_once() - @pytest.mark.asyncio - async def test_validate(self, mock_async_client: AsyncMock) -> None: - """Test validate method.""" - validate_response = SimpleNamespace( - name="test_scorer", - scoring_context={}, - scoring_result=SimpleNamespace(score=0.95), - ) - mock_async_client.scenarios.scorers.validate = AsyncMock(return_value=validate_response) - - scorer = AsyncScorer(mock_async_client, "sco_123") - result = await scorer.validate( - scoring_context={"test": "context"}, - ) - - assert result == validate_response - mock_async_client.scenarios.scorers.validate.assert_awaited_once() diff --git a/tests/sdk/test_scenario_builder.py b/tests/sdk/test_scenario_builder.py index 75597d5f3..cc9a94707 100644 --- a/tests/sdk/test_scenario_builder.py +++ b/tests/sdk/test_scenario_builder.py @@ -107,16 +107,8 @@ def test_scorers(self, mock_builder: ScenarioBuilder) -> None: assert mock_builder._scorers[4]["scorer"].get("pattern") == "$A.foo()" assert mock_builder._scorers[4]["scorer"].get("lang") == "python" - # Custom scorer with optional params - mock_builder.add_custom_scorer( - "custom-scorer", custom_scorer_type="my_scorer", scorer_params={"threshold": 0.5} - ) - assert mock_builder._scorers[5]["scorer"]["type"] == "custom_scorer" - assert mock_builder._scorers[5]["scorer"].get("custom_scorer_type") == "my_scorer" - assert mock_builder._scorers[5]["scorer"].get("scorer_params") == {"threshold": 0.5} - # Verify multiple scorers accumulated - assert len(mock_builder._scorers) == 6 + assert len(mock_builder._scorers) == 5 def test_add_scorer_rejects_invalid_weight(self, mock_builder: ScenarioBuilder) -> None: """Test that adding a scorer with zero or negative weight raises ValueError.""" diff --git a/tests/sdk/test_scorer.py b/tests/sdk/test_scorer.py index 91b430db0..3a91867ab 100644 --- a/tests/sdk/test_scorer.py +++ b/tests/sdk/test_scorer.py @@ -50,22 +50,3 @@ def test_update(self, mock_client: Mock) -> None: bash_script="echo 'score=1.0'", ) - def test_validate(self, mock_client: Mock) -> None: - """Test validate method.""" - validate_response = SimpleNamespace( - name="test_scorer", - scoring_context={}, - scoring_result=SimpleNamespace(score=0.95), - ) - mock_client.scenarios.scorers.validate.return_value = validate_response - - scorer = Scorer(mock_client, "sco_123") - result = scorer.validate( - scoring_context={"test": "context"}, - ) - - assert result == validate_response - mock_client.scenarios.scorers.validate.assert_called_once_with( - "sco_123", - scoring_context={"test": "context"}, - ) From feeb54c3ff27a682966982790361225d9ce138a7 Mon Sep 17 00:00:00 2001 From: James Chainey Date: Wed, 4 Feb 2026 17:33:59 -0800 Subject: [PATCH 2/3] fmt --- tests/sdk/test_async_scorer.py | 1 - tests/sdk/test_scorer.py | 1 - 2 files changed, 2 deletions(-) diff --git a/tests/sdk/test_async_scorer.py b/tests/sdk/test_async_scorer.py index 637bba044..23b6628c7 100644 --- a/tests/sdk/test_async_scorer.py +++ b/tests/sdk/test_async_scorer.py @@ -49,4 +49,3 @@ async def test_update(self, mock_async_client: AsyncMock) -> None: assert result == update_response mock_async_client.scenarios.scorers.update.assert_awaited_once() - diff --git a/tests/sdk/test_scorer.py b/tests/sdk/test_scorer.py index 3a91867ab..e3363c6dc 100644 --- a/tests/sdk/test_scorer.py +++ b/tests/sdk/test_scorer.py @@ -49,4 +49,3 @@ def test_update(self, mock_client: Mock) -> None: type="updated_scorer", bash_script="echo 'score=1.0'", ) - From 1f5d8a3e710f732a1cde30a391905346a0ea1482 Mon Sep 17 00:00:00 2001 From: James Chainey Date: Wed, 4 Feb 2026 17:53:29 -0800 Subject: [PATCH 3/3] removed dead tests --- tests/api_resources/scenarios/test_scorers.py | 151 ------------------ 1 file changed, 151 deletions(-) diff --git a/tests/api_resources/scenarios/test_scorers.py b/tests/api_resources/scenarios/test_scorers.py index 73be902fd..359e0dcc7 100644 --- a/tests/api_resources/scenarios/test_scorers.py +++ b/tests/api_resources/scenarios/test_scorers.py @@ -15,7 +15,6 @@ ScorerCreateResponse, ScorerUpdateResponse, ScorerRetrieveResponse, - ScorerValidateResponse, ) base_url = os.environ.get("TEST_API_BASE_URL", "http://127.0.0.1:4010") @@ -175,81 +174,6 @@ def test_streaming_response_list(self, client: Runloop) -> None: assert cast(Any, response.is_closed) is True - @parametrize - def test_method_validate(self, client: Runloop) -> None: - scorer = client.scenarios.scorers.validate( - id="id", - scoring_context={}, - ) - assert_matches_type(ScorerValidateResponse, scorer, path=["response"]) - - @parametrize - def test_method_validate_with_all_params(self, client: Runloop) -> None: - scorer = client.scenarios.scorers.validate( - id="id", - scoring_context={}, - environment_parameters={ - "blueprint_id": "blueprint_id", - "launch_parameters": { - "after_idle": { - "idle_time_seconds": 0, - "on_idle": "shutdown", - }, - "architecture": "x86_64", - "available_ports": [0], - "custom_cpu_cores": 0, - "custom_disk_size": 0, - "custom_gb_memory": 0, - "keep_alive_time_seconds": 0, - "launch_commands": ["string"], - "network_policy_id": "network_policy_id", - "required_services": ["string"], - "resource_size_request": "X_SMALL", - "user_parameters": { - "uid": 0, - "username": "username", - }, - }, - "snapshot_id": "snapshot_id", - "working_directory": "working_directory", - }, - ) - assert_matches_type(ScorerValidateResponse, scorer, path=["response"]) - - @parametrize - def test_raw_response_validate(self, client: Runloop) -> None: - response = client.scenarios.scorers.with_raw_response.validate( - id="id", - scoring_context={}, - ) - - assert response.is_closed is True - assert response.http_request.headers.get("X-Stainless-Lang") == "python" - scorer = response.parse() - assert_matches_type(ScorerValidateResponse, scorer, path=["response"]) - - @parametrize - def test_streaming_response_validate(self, client: Runloop) -> None: - with client.scenarios.scorers.with_streaming_response.validate( - id="id", - scoring_context={}, - ) as response: - assert not response.is_closed - assert response.http_request.headers.get("X-Stainless-Lang") == "python" - - scorer = response.parse() - assert_matches_type(ScorerValidateResponse, scorer, path=["response"]) - - assert cast(Any, response.is_closed) is True - - @parametrize - def test_path_params_validate(self, client: Runloop) -> None: - with pytest.raises(ValueError, match=r"Expected a non-empty value for `id` but received ''"): - client.scenarios.scorers.with_raw_response.validate( - id="", - scoring_context={}, - ) - class TestAsyncScorers: parametrize = pytest.mark.parametrize( @@ -406,78 +330,3 @@ async def test_streaming_response_list(self, async_client: AsyncRunloop) -> None assert_matches_type(AsyncScenarioScorersCursorIDPage[ScorerListResponse], scorer, path=["response"]) assert cast(Any, response.is_closed) is True - - @parametrize - async def test_method_validate(self, async_client: AsyncRunloop) -> None: - scorer = await async_client.scenarios.scorers.validate( - id="id", - scoring_context={}, - ) - assert_matches_type(ScorerValidateResponse, scorer, path=["response"]) - - @parametrize - async def test_method_validate_with_all_params(self, async_client: AsyncRunloop) -> None: - scorer = await async_client.scenarios.scorers.validate( - id="id", - scoring_context={}, - environment_parameters={ - "blueprint_id": "blueprint_id", - "launch_parameters": { - "after_idle": { - "idle_time_seconds": 0, - "on_idle": "shutdown", - }, - "architecture": "x86_64", - "available_ports": [0], - "custom_cpu_cores": 0, - "custom_disk_size": 0, - "custom_gb_memory": 0, - "keep_alive_time_seconds": 0, - "launch_commands": ["string"], - "network_policy_id": "network_policy_id", - "required_services": ["string"], - "resource_size_request": "X_SMALL", - "user_parameters": { - "uid": 0, - "username": "username", - }, - }, - "snapshot_id": "snapshot_id", - "working_directory": "working_directory", - }, - ) - assert_matches_type(ScorerValidateResponse, scorer, path=["response"]) - - @parametrize - async def test_raw_response_validate(self, async_client: AsyncRunloop) -> None: - response = await async_client.scenarios.scorers.with_raw_response.validate( - id="id", - scoring_context={}, - ) - - assert response.is_closed is True - assert response.http_request.headers.get("X-Stainless-Lang") == "python" - scorer = await response.parse() - assert_matches_type(ScorerValidateResponse, scorer, path=["response"]) - - @parametrize - async def test_streaming_response_validate(self, async_client: AsyncRunloop) -> None: - async with async_client.scenarios.scorers.with_streaming_response.validate( - id="id", - scoring_context={}, - ) as response: - assert not response.is_closed - assert response.http_request.headers.get("X-Stainless-Lang") == "python" - - scorer = await response.parse() - assert_matches_type(ScorerValidateResponse, scorer, path=["response"]) - - assert cast(Any, response.is_closed) is True - - @parametrize - async def test_path_params_validate(self, async_client: AsyncRunloop) -> None: - with pytest.raises(ValueError, match=r"Expected a non-empty value for `id` but received ''"): - await async_client.scenarios.scorers.with_raw_response.validate( - id="", - scoring_context={}, - )