From bedb9a7b8839e193107e88b92f7cc166f08ac83d Mon Sep 17 00:00:00 2001 From: Alex Chang Date: Fri, 21 Aug 2026 09:29:57 -0400 Subject: [PATCH 1/5] chore(api): Clarify image background docs and preview support (#3703) ## Summary Refine documentation for the image 'background' parameter across the Images resource and related type definitions. These changes only update descriptions and wording; there are no changes to function signatures, types, or runtime behavior. ## Changes - Rewrite and clarify the 'background' parameter description for image generation and editing to state it must be one of 'transparent', 'opaque', or 'auto' and that 'auto' lets the model choose the background. - Document that transparent backgrounds are available for supported GPT Image models and explicitly mark gpt-image-2 and gpt-image-2-2026-04-21 support as preview-level, rather than stating transparent backgrounds are not supported. - Add/clarify guidance that when requesting a transparent background the returned image format should support transparency (e.g., 'png' or 'webp'). - Apply these documentation updates consistently in runtime resource docstrings and in typed models/typed dicts describing image parameters and responses. - No changes to public types, method signatures, parameter names, accepted values, or return types; no customer-visible behavior or API contract changes. Co-authored-by: apcha-oai <228803254+apcha-oai@users.noreply.github.com> --- .castiron.stats.yml | 14 +- api_reference/openapi.transformed.yml | 87 ++++----- src/openai/resources/images.py | 204 ++++++++-------------- src/openai/types/beta/beta_tool.py | 19 +- src/openai/types/beta/beta_tool_param.py | 19 +- src/openai/types/image_edit_params.py | 19 +- src/openai/types/image_generate_params.py | 19 +- src/openai/types/responses/tool.py | 19 +- src/openai/types/responses/tool_param.py | 19 +- 9 files changed, 157 insertions(+), 262 deletions(-) diff --git a/.castiron.stats.yml b/.castiron.stats.yml index 26d99d11aa..e32ac6fcbd 100644 --- a/.castiron.stats.yml +++ b/.castiron.stats.yml @@ -1,8 +1,8 @@ schema_version: 1 -generation_id: ddf51c0b-5978-487f-be21-e379112a927a -openapi_spec_hash: a85edbfc22ff719d064bce2705c7394e -openapi_transformed_spec_hash: f8e7644df5aee22dfcd0ea2b70942054 -config_hash: 85382dd94c503b5d225adc7636a77c9f -codegen_sha: 310aa46b5b69a9e4a3d0dd78fa47797d6cae2746 -codegen_hash: 5188f6aac875d009719a2f3702c2824068b7ce0780fe001f8a791735d8212a8b -public_codegen_sha: 0d3e70da47bb645fbfd0dd16fdd37ea10ee981f5 +generation_id: a346904e-ba2c-4683-831d-a452ad321113 +openapi_spec_hash: 92700e1a33a4f6174a01b46648c8186a +openapi_transformed_spec_hash: e37bbe0f04caa6093f1cb5d65d23ad03 +config_hash: d13c582815d0db08c374985806be7352 +codegen_sha: 035b803d39b5d87e9ea7624fdf638f58ff6343e9 +codegen_hash: 015182335504f95ee327a5096cdb3c6046b558c54ddac2ddb9d6b526be4b536d +public_codegen_sha: e36749421d1be152d016bc41a8850bbdd9de0e63 diff --git a/api_reference/openapi.transformed.yml b/api_reference/openapi.transformed.yml index f14d867a01..c12baba61f 100644 --- a/api_reference/openapi.transformed.yml +++ b/api_reference/openapi.transformed.yml @@ -33071,18 +33071,14 @@ components: nullable: true description: | Allows to set transparency for the background of the generated image(s). - This parameter is only supported for GPT image models that support - transparent backgrounds. Must be one of `transparent`, `opaque`, or - `auto` (default value). When `auto` is used, the model will - automatically determine the best background for the image. - - `gpt-image-2` and `gpt-image-2-2026-04-21` do not support - transparent backgrounds. Requests with `background` set to - `transparent` will return an error for these models; use `opaque` or - `auto` instead. - - If `transparent`, the output format needs to support transparency, - so it should be set to either `png` (default value) or `webp`. + Must be one of `transparent`, `opaque`, or `auto` (default value). When + `auto` is used, the model will automatically determine the best + background for the image. + + Transparent backgrounds are available for supported GPT Image models. + For `gpt-image-2` and `gpt-image-2-2026-04-21`, this support is in + preview. When using `transparent`, set the output format to `png` or + `webp`. model: anyOf: - type: string @@ -33310,18 +33306,14 @@ components: nullable: true description: | Allows to set transparency for the background of the generated image(s). - This parameter is only supported for GPT image models that support - transparent backgrounds. Must be one of `transparent`, `opaque`, or - `auto` (default value). When `auto` is used, the model will - automatically determine the best background for the image. - - `gpt-image-2` and `gpt-image-2-2026-04-21` do not support - transparent backgrounds. Requests with `background` set to - `transparent` will return an error for these models; use `opaque` or - `auto` instead. - - If `transparent`, the output format needs to support transparency, - so it should be set to either `png` (default value) or `webp`. + Must be one of `transparent`, `opaque`, or `auto` (default value). When + `auto` is used, the model will automatically determine the best + background for the image. + + Transparent backgrounds are available for supported GPT Image models. + For `gpt-image-2` and `gpt-image-2-2026-04-21`, this support is in + preview. When using `transparent`, set the output format to `png` or + `webp`. style: type: string enum: @@ -35512,7 +35504,7 @@ components: - type: 'null' default: auto example: transparent - description: Background behavior for generated image output. + description: Set the background of the generated image output. Transparent backgrounds are available for supported GPT Image models. For `gpt-image-2` and `gpt-image-2-2026-04-21`, this support is in preview. When using `transparent`, set the output format to `png` or `webp`. stream: anyOf: - type: boolean @@ -39198,18 +39190,14 @@ components: - auto description: | Allows to set transparency for the background of the generated image(s). - This parameter is only supported for GPT image models that support - transparent backgrounds. Must be one of `transparent`, `opaque`, or - `auto` (default value). When `auto` is used, the model will - automatically determine the best background for the image. - - `gpt-image-2` and `gpt-image-2-2026-04-21` do not support - transparent backgrounds. Requests with `background` set to - `transparent` will return an error for these models; use `opaque` or - `auto` instead. - - If `transparent`, the output format needs to support transparency, - so it should be set to either `png` (default value) or `webp`. + Must be one of `transparent`, `opaque`, or `auto` (default value). When + `auto` is used, the model will automatically determine the best + background for the image. + + Transparent backgrounds are available for supported GPT Image models. + For `gpt-image-2` and `gpt-image-2-2026-04-21`, this support is in + preview. When using `transparent`, set the output format to `png` or + `webp`. default: auto input_fidelity: anyOf: @@ -51417,8 +51405,9 @@ components: RealtimeTranslationServerEventSessionOutputAudioDelta: type: object description: | - Returned when translated output audio is available. Output audio deltas are - 200 ms frames of PCM16 audio. + Returned when translated output audio is available. The `delta` contains a + PCM16 audio chunk whose length can vary. Clients should decode and queue the + complete delta instead of assuming a fixed byte or sample count. properties: event_id: type: string @@ -68577,18 +68566,14 @@ components: - auto description: | Allows to set transparency for the background of the generated image(s). - This parameter is only supported for GPT image models that support - transparent backgrounds. Must be one of `transparent`, `opaque`, or - `auto` (default value). When `auto` is used, the model will - automatically determine the best background for the image. - - `gpt-image-2` and `gpt-image-2-2026-04-21` do not support - transparent backgrounds. Requests with `background` set to - `transparent` will return an error for these models; use `opaque` or - `auto` instead. - - If `transparent`, the output format needs to support transparency, - so it should be set to either `png` (default value) or `webp`. + Must be one of `transparent`, `opaque`, or `auto` (default value). When + `auto` is used, the model will automatically determine the best + background for the image. + + Transparent backgrounds are available for supported GPT Image models. + For `gpt-image-2` and `gpt-image-2-2026-04-21`, this support is in + preview. When using `transparent`, set the output format to `png` or + `webp`. default: auto input_fidelity: anyOf: diff --git a/src/openai/resources/images.py b/src/openai/resources/images.py index e0aad166fc..8081f3532c 100644 --- a/src/openai/resources/images.py +++ b/src/openai/resources/images.py @@ -172,18 +172,13 @@ def edit( prompt: A text description of the desired image(s). The maximum length is 1000 characters for `dall-e-2`, and 32000 characters for the GPT image models. - background: Allows to set transparency for the background of the generated image(s). This - parameter is only supported for GPT image models that support transparent - backgrounds. Must be one of `transparent`, `opaque`, or `auto` (default value). - When `auto` is used, the model will automatically determine the best background - for the image. + background: Allows to set transparency for the background of the generated image(s). Must be + one of `transparent`, `opaque`, or `auto` (default value). When `auto` is used, + the model will automatically determine the best background for the image. - `gpt-image-2` and `gpt-image-2-2026-04-21` do not support transparent - backgrounds. Requests with `background` set to `transparent` will return an - error for these models; use `opaque` or `auto` instead. - - If `transparent`, the output format needs to support transparency, so it should - be set to either `png` (default value) or `webp`. + Transparent backgrounds are available for supported GPT Image models. For + `gpt-image-2` and `gpt-image-2-2026-04-21`, this support is in preview. When + using `transparent`, set the output format to `png` or `webp`. input_fidelity: Control how much effort the model will exert to match the style and features, especially facial features, of input images. This parameter is only supported @@ -306,18 +301,13 @@ def edit( [Image generation guide](https://platform.openai.com/docs/guides/image-generation) for more information. - background: Allows to set transparency for the background of the generated image(s). This - parameter is only supported for GPT image models that support transparent - backgrounds. Must be one of `transparent`, `opaque`, or `auto` (default value). - When `auto` is used, the model will automatically determine the best background - for the image. - - `gpt-image-2` and `gpt-image-2-2026-04-21` do not support transparent - backgrounds. Requests with `background` set to `transparent` will return an - error for these models; use `opaque` or `auto` instead. + background: Allows to set transparency for the background of the generated image(s). Must be + one of `transparent`, `opaque`, or `auto` (default value). When `auto` is used, + the model will automatically determine the best background for the image. - If `transparent`, the output format needs to support transparency, so it should - be set to either `png` (default value) or `webp`. + Transparent backgrounds are available for supported GPT Image models. For + `gpt-image-2` and `gpt-image-2-2026-04-21`, this support is in preview. When + using `transparent`, set the output format to `png` or `webp`. input_fidelity: Control how much effort the model will exert to match the style and features, especially facial features, of input images. This parameter is only supported @@ -436,18 +426,13 @@ def edit( [Image generation guide](https://platform.openai.com/docs/guides/image-generation) for more information. - background: Allows to set transparency for the background of the generated image(s). This - parameter is only supported for GPT image models that support transparent - backgrounds. Must be one of `transparent`, `opaque`, or `auto` (default value). - When `auto` is used, the model will automatically determine the best background - for the image. - - `gpt-image-2` and `gpt-image-2-2026-04-21` do not support transparent - backgrounds. Requests with `background` set to `transparent` will return an - error for these models; use `opaque` or `auto` instead. + background: Allows to set transparency for the background of the generated image(s). Must be + one of `transparent`, `opaque`, or `auto` (default value). When `auto` is used, + the model will automatically determine the best background for the image. - If `transparent`, the output format needs to support transparency, so it should - be set to either `png` (default value) or `webp`. + Transparent backgrounds are available for supported GPT Image models. For + `gpt-image-2` and `gpt-image-2-2026-04-21`, this support is in preview. When + using `transparent`, set the output format to `png` or `webp`. input_fidelity: Control how much effort the model will exert to match the style and features, especially facial features, of input images. This parameter is only supported @@ -625,18 +610,13 @@ def generate( characters for the GPT image models, 1000 characters for `dall-e-2` and 4000 characters for `dall-e-3`. - background: Allows to set transparency for the background of the generated image(s). This - parameter is only supported for GPT image models that support transparent - backgrounds. Must be one of `transparent`, `opaque`, or `auto` (default value). - When `auto` is used, the model will automatically determine the best background - for the image. + background: Allows to set transparency for the background of the generated image(s). Must be + one of `transparent`, `opaque`, or `auto` (default value). When `auto` is used, + the model will automatically determine the best background for the image. - `gpt-image-2` and `gpt-image-2-2026-04-21` do not support transparent - backgrounds. Requests with `background` set to `transparent` will return an - error for these models; use `opaque` or `auto` instead. - - If `transparent`, the output format needs to support transparency, so it should - be set to either `png` (default value) or `webp`. + Transparent backgrounds are available for supported GPT Image models. For + `gpt-image-2` and `gpt-image-2-2026-04-21`, this support is in preview. When + using `transparent`, set the output format to `png` or `webp`. model: The model to use for image generation. One of `dall-e-2`, `dall-e-3`, or a GPT image model (`gpt-image-1`, `gpt-image-1-mini`, `gpt-image-1.5`, `gpt-image-2`, @@ -755,18 +735,13 @@ def generate( [Image generation guide](https://platform.openai.com/docs/guides/image-generation) for more information. This parameter is only supported for the GPT image models. - background: Allows to set transparency for the background of the generated image(s). This - parameter is only supported for GPT image models that support transparent - backgrounds. Must be one of `transparent`, `opaque`, or `auto` (default value). - When `auto` is used, the model will automatically determine the best background - for the image. - - `gpt-image-2` and `gpt-image-2-2026-04-21` do not support transparent - backgrounds. Requests with `background` set to `transparent` will return an - error for these models; use `opaque` or `auto` instead. + background: Allows to set transparency for the background of the generated image(s). Must be + one of `transparent`, `opaque`, or `auto` (default value). When `auto` is used, + the model will automatically determine the best background for the image. - If `transparent`, the output format needs to support transparency, so it should - be set to either `png` (default value) or `webp`. + Transparent backgrounds are available for supported GPT Image models. For + `gpt-image-2` and `gpt-image-2-2026-04-21`, this support is in preview. When + using `transparent`, set the output format to `png` or `webp`. model: The model to use for image generation. One of `dall-e-2`, `dall-e-3`, or a GPT image model (`gpt-image-1`, `gpt-image-1-mini`, `gpt-image-1.5`, `gpt-image-2`, @@ -881,18 +856,13 @@ def generate( [Image generation guide](https://platform.openai.com/docs/guides/image-generation) for more information. This parameter is only supported for the GPT image models. - background: Allows to set transparency for the background of the generated image(s). This - parameter is only supported for GPT image models that support transparent - backgrounds. Must be one of `transparent`, `opaque`, or `auto` (default value). - When `auto` is used, the model will automatically determine the best background - for the image. - - `gpt-image-2` and `gpt-image-2-2026-04-21` do not support transparent - backgrounds. Requests with `background` set to `transparent` will return an - error for these models; use `opaque` or `auto` instead. + background: Allows to set transparency for the background of the generated image(s). Must be + one of `transparent`, `opaque`, or `auto` (default value). When `auto` is used, + the model will automatically determine the best background for the image. - If `transparent`, the output format needs to support transparency, so it should - be set to either `png` (default value) or `webp`. + Transparent backgrounds are available for supported GPT Image models. For + `gpt-image-2` and `gpt-image-2-2026-04-21`, this support is in preview. When + using `transparent`, set the output format to `png` or `webp`. model: The model to use for image generation. One of `dall-e-2`, `dall-e-3`, or a GPT image model (`gpt-image-1`, `gpt-image-1-mini`, `gpt-image-1.5`, `gpt-image-2`, @@ -1177,18 +1147,13 @@ async def edit( prompt: A text description of the desired image(s). The maximum length is 1000 characters for `dall-e-2`, and 32000 characters for the GPT image models. - background: Allows to set transparency for the background of the generated image(s). This - parameter is only supported for GPT image models that support transparent - backgrounds. Must be one of `transparent`, `opaque`, or `auto` (default value). - When `auto` is used, the model will automatically determine the best background - for the image. + background: Allows to set transparency for the background of the generated image(s). Must be + one of `transparent`, `opaque`, or `auto` (default value). When `auto` is used, + the model will automatically determine the best background for the image. - `gpt-image-2` and `gpt-image-2-2026-04-21` do not support transparent - backgrounds. Requests with `background` set to `transparent` will return an - error for these models; use `opaque` or `auto` instead. - - If `transparent`, the output format needs to support transparency, so it should - be set to either `png` (default value) or `webp`. + Transparent backgrounds are available for supported GPT Image models. For + `gpt-image-2` and `gpt-image-2-2026-04-21`, this support is in preview. When + using `transparent`, set the output format to `png` or `webp`. input_fidelity: Control how much effort the model will exert to match the style and features, especially facial features, of input images. This parameter is only supported @@ -1311,18 +1276,13 @@ async def edit( [Image generation guide](https://platform.openai.com/docs/guides/image-generation) for more information. - background: Allows to set transparency for the background of the generated image(s). This - parameter is only supported for GPT image models that support transparent - backgrounds. Must be one of `transparent`, `opaque`, or `auto` (default value). - When `auto` is used, the model will automatically determine the best background - for the image. - - `gpt-image-2` and `gpt-image-2-2026-04-21` do not support transparent - backgrounds. Requests with `background` set to `transparent` will return an - error for these models; use `opaque` or `auto` instead. + background: Allows to set transparency for the background of the generated image(s). Must be + one of `transparent`, `opaque`, or `auto` (default value). When `auto` is used, + the model will automatically determine the best background for the image. - If `transparent`, the output format needs to support transparency, so it should - be set to either `png` (default value) or `webp`. + Transparent backgrounds are available for supported GPT Image models. For + `gpt-image-2` and `gpt-image-2-2026-04-21`, this support is in preview. When + using `transparent`, set the output format to `png` or `webp`. input_fidelity: Control how much effort the model will exert to match the style and features, especially facial features, of input images. This parameter is only supported @@ -1441,18 +1401,13 @@ async def edit( [Image generation guide](https://platform.openai.com/docs/guides/image-generation) for more information. - background: Allows to set transparency for the background of the generated image(s). This - parameter is only supported for GPT image models that support transparent - backgrounds. Must be one of `transparent`, `opaque`, or `auto` (default value). - When `auto` is used, the model will automatically determine the best background - for the image. - - `gpt-image-2` and `gpt-image-2-2026-04-21` do not support transparent - backgrounds. Requests with `background` set to `transparent` will return an - error for these models; use `opaque` or `auto` instead. + background: Allows to set transparency for the background of the generated image(s). Must be + one of `transparent`, `opaque`, or `auto` (default value). When `auto` is used, + the model will automatically determine the best background for the image. - If `transparent`, the output format needs to support transparency, so it should - be set to either `png` (default value) or `webp`. + Transparent backgrounds are available for supported GPT Image models. For + `gpt-image-2` and `gpt-image-2-2026-04-21`, this support is in preview. When + using `transparent`, set the output format to `png` or `webp`. input_fidelity: Control how much effort the model will exert to match the style and features, especially facial features, of input images. This parameter is only supported @@ -1630,18 +1585,13 @@ async def generate( characters for the GPT image models, 1000 characters for `dall-e-2` and 4000 characters for `dall-e-3`. - background: Allows to set transparency for the background of the generated image(s). This - parameter is only supported for GPT image models that support transparent - backgrounds. Must be one of `transparent`, `opaque`, or `auto` (default value). - When `auto` is used, the model will automatically determine the best background - for the image. + background: Allows to set transparency for the background of the generated image(s). Must be + one of `transparent`, `opaque`, or `auto` (default value). When `auto` is used, + the model will automatically determine the best background for the image. - `gpt-image-2` and `gpt-image-2-2026-04-21` do not support transparent - backgrounds. Requests with `background` set to `transparent` will return an - error for these models; use `opaque` or `auto` instead. - - If `transparent`, the output format needs to support transparency, so it should - be set to either `png` (default value) or `webp`. + Transparent backgrounds are available for supported GPT Image models. For + `gpt-image-2` and `gpt-image-2-2026-04-21`, this support is in preview. When + using `transparent`, set the output format to `png` or `webp`. model: The model to use for image generation. One of `dall-e-2`, `dall-e-3`, or a GPT image model (`gpt-image-1`, `gpt-image-1-mini`, `gpt-image-1.5`, `gpt-image-2`, @@ -1760,18 +1710,13 @@ async def generate( [Image generation guide](https://platform.openai.com/docs/guides/image-generation) for more information. This parameter is only supported for the GPT image models. - background: Allows to set transparency for the background of the generated image(s). This - parameter is only supported for GPT image models that support transparent - backgrounds. Must be one of `transparent`, `opaque`, or `auto` (default value). - When `auto` is used, the model will automatically determine the best background - for the image. - - `gpt-image-2` and `gpt-image-2-2026-04-21` do not support transparent - backgrounds. Requests with `background` set to `transparent` will return an - error for these models; use `opaque` or `auto` instead. + background: Allows to set transparency for the background of the generated image(s). Must be + one of `transparent`, `opaque`, or `auto` (default value). When `auto` is used, + the model will automatically determine the best background for the image. - If `transparent`, the output format needs to support transparency, so it should - be set to either `png` (default value) or `webp`. + Transparent backgrounds are available for supported GPT Image models. For + `gpt-image-2` and `gpt-image-2-2026-04-21`, this support is in preview. When + using `transparent`, set the output format to `png` or `webp`. model: The model to use for image generation. One of `dall-e-2`, `dall-e-3`, or a GPT image model (`gpt-image-1`, `gpt-image-1-mini`, `gpt-image-1.5`, `gpt-image-2`, @@ -1886,18 +1831,13 @@ async def generate( [Image generation guide](https://platform.openai.com/docs/guides/image-generation) for more information. This parameter is only supported for the GPT image models. - background: Allows to set transparency for the background of the generated image(s). This - parameter is only supported for GPT image models that support transparent - backgrounds. Must be one of `transparent`, `opaque`, or `auto` (default value). - When `auto` is used, the model will automatically determine the best background - for the image. - - `gpt-image-2` and `gpt-image-2-2026-04-21` do not support transparent - backgrounds. Requests with `background` set to `transparent` will return an - error for these models; use `opaque` or `auto` instead. + background: Allows to set transparency for the background of the generated image(s). Must be + one of `transparent`, `opaque`, or `auto` (default value). When `auto` is used, + the model will automatically determine the best background for the image. - If `transparent`, the output format needs to support transparency, so it should - be set to either `png` (default value) or `webp`. + Transparent backgrounds are available for supported GPT Image models. For + `gpt-image-2` and `gpt-image-2-2026-04-21`, this support is in preview. When + using `transparent`, set the output format to `png` or `webp`. model: The model to use for image generation. One of `dall-e-2`, `dall-e-3`, or a GPT image model (`gpt-image-1`, `gpt-image-1-mini`, `gpt-image-1.5`, `gpt-image-2`, diff --git a/src/openai/types/beta/beta_tool.py b/src/openai/types/beta/beta_tool.py index 57c659c967..faa0ed24ee 100644 --- a/src/openai/types/beta/beta_tool.py +++ b/src/openai/types/beta/beta_tool.py @@ -262,18 +262,13 @@ class ImageGeneration(BaseModel): background: Optional[Literal["transparent", "opaque", "auto"]] = None """ - Allows to set transparency for the background of the generated image(s). This - parameter is only supported for GPT image models that support transparent - backgrounds. Must be one of `transparent`, `opaque`, or `auto` (default value). - When `auto` is used, the model will automatically determine the best background - for the image. - - `gpt-image-2` and `gpt-image-2-2026-04-21` do not support transparent - backgrounds. Requests with `background` set to `transparent` will return an - error for these models; use `opaque` or `auto` instead. - - If `transparent`, the output format needs to support transparency, so it should - be set to either `png` (default value) or `webp`. + Allows to set transparency for the background of the generated image(s). Must be + one of `transparent`, `opaque`, or `auto` (default value). When `auto` is used, + the model will automatically determine the best background for the image. + + Transparent backgrounds are available for supported GPT Image models. For + `gpt-image-2` and `gpt-image-2-2026-04-21`, this support is in preview. When + using `transparent`, set the output format to `png` or `webp`. """ input_fidelity: Optional[Literal["high", "low"]] = None diff --git a/src/openai/types/beta/beta_tool_param.py b/src/openai/types/beta/beta_tool_param.py index 640b4bfedb..86527185d2 100644 --- a/src/openai/types/beta/beta_tool_param.py +++ b/src/openai/types/beta/beta_tool_param.py @@ -261,18 +261,13 @@ class ImageGeneration(TypedDict, total=False): background: Literal["transparent", "opaque", "auto"] """ - Allows to set transparency for the background of the generated image(s). This - parameter is only supported for GPT image models that support transparent - backgrounds. Must be one of `transparent`, `opaque`, or `auto` (default value). - When `auto` is used, the model will automatically determine the best background - for the image. - - `gpt-image-2` and `gpt-image-2-2026-04-21` do not support transparent - backgrounds. Requests with `background` set to `transparent` will return an - error for these models; use `opaque` or `auto` instead. - - If `transparent`, the output format needs to support transparency, so it should - be set to either `png` (default value) or `webp`. + Allows to set transparency for the background of the generated image(s). Must be + one of `transparent`, `opaque`, or `auto` (default value). When `auto` is used, + the model will automatically determine the best background for the image. + + Transparent backgrounds are available for supported GPT Image models. For + `gpt-image-2` and `gpt-image-2-2026-04-21`, this support is in preview. When + using `transparent`, set the output format to `png` or `webp`. """ input_fidelity: Optional[Literal["high", "low"]] diff --git a/src/openai/types/image_edit_params.py b/src/openai/types/image_edit_params.py index b014454606..2f691d2795 100644 --- a/src/openai/types/image_edit_params.py +++ b/src/openai/types/image_edit_params.py @@ -33,18 +33,13 @@ class ImageEditParamsBase(TypedDict, total=False): background: Optional[Literal["transparent", "opaque", "auto"]] """ - Allows to set transparency for the background of the generated image(s). This - parameter is only supported for GPT image models that support transparent - backgrounds. Must be one of `transparent`, `opaque`, or `auto` (default value). - When `auto` is used, the model will automatically determine the best background - for the image. - - `gpt-image-2` and `gpt-image-2-2026-04-21` do not support transparent - backgrounds. Requests with `background` set to `transparent` will return an - error for these models; use `opaque` or `auto` instead. - - If `transparent`, the output format needs to support transparency, so it should - be set to either `png` (default value) or `webp`. + Allows to set transparency for the background of the generated image(s). Must be + one of `transparent`, `opaque`, or `auto` (default value). When `auto` is used, + the model will automatically determine the best background for the image. + + Transparent backgrounds are available for supported GPT Image models. For + `gpt-image-2` and `gpt-image-2-2026-04-21`, this support is in preview. When + using `transparent`, set the output format to `png` or `webp`. """ input_fidelity: Optional[Literal["high", "low"]] diff --git a/src/openai/types/image_generate_params.py b/src/openai/types/image_generate_params.py index c9aeacd30b..59cf4c191f 100644 --- a/src/openai/types/image_generate_params.py +++ b/src/openai/types/image_generate_params.py @@ -20,18 +20,13 @@ class ImageGenerateParamsBase(TypedDict, total=False): background: Optional[Literal["transparent", "opaque", "auto"]] """ - Allows to set transparency for the background of the generated image(s). This - parameter is only supported for GPT image models that support transparent - backgrounds. Must be one of `transparent`, `opaque`, or `auto` (default value). - When `auto` is used, the model will automatically determine the best background - for the image. - - `gpt-image-2` and `gpt-image-2-2026-04-21` do not support transparent - backgrounds. Requests with `background` set to `transparent` will return an - error for these models; use `opaque` or `auto` instead. - - If `transparent`, the output format needs to support transparency, so it should - be set to either `png` (default value) or `webp`. + Allows to set transparency for the background of the generated image(s). Must be + one of `transparent`, `opaque`, or `auto` (default value). When `auto` is used, + the model will automatically determine the best background for the image. + + Transparent backgrounds are available for supported GPT Image models. For + `gpt-image-2` and `gpt-image-2-2026-04-21`, this support is in preview. When + using `transparent`, set the output format to `png` or `webp`. """ model: Union[str, ImageModel, None] diff --git a/src/openai/types/responses/tool.py b/src/openai/types/responses/tool.py index 69293fa845..3e815cceff 100644 --- a/src/openai/types/responses/tool.py +++ b/src/openai/types/responses/tool.py @@ -267,18 +267,13 @@ class ImageGeneration(BaseModel): background: Optional[Literal["transparent", "opaque", "auto"]] = None """ - Allows to set transparency for the background of the generated image(s). This - parameter is only supported for GPT image models that support transparent - backgrounds. Must be one of `transparent`, `opaque`, or `auto` (default value). - When `auto` is used, the model will automatically determine the best background - for the image. - - `gpt-image-2` and `gpt-image-2-2026-04-21` do not support transparent - backgrounds. Requests with `background` set to `transparent` will return an - error for these models; use `opaque` or `auto` instead. - - If `transparent`, the output format needs to support transparency, so it should - be set to either `png` (default value) or `webp`. + Allows to set transparency for the background of the generated image(s). Must be + one of `transparent`, `opaque`, or `auto` (default value). When `auto` is used, + the model will automatically determine the best background for the image. + + Transparent backgrounds are available for supported GPT Image models. For + `gpt-image-2` and `gpt-image-2-2026-04-21`, this support is in preview. When + using `transparent`, set the output format to `png` or `webp`. """ input_fidelity: Optional[Literal["high", "low"]] = None diff --git a/src/openai/types/responses/tool_param.py b/src/openai/types/responses/tool_param.py index 9aa229cdd9..a1fb90c8c3 100644 --- a/src/openai/types/responses/tool_param.py +++ b/src/openai/types/responses/tool_param.py @@ -267,18 +267,13 @@ class ImageGeneration(TypedDict, total=False): background: Literal["transparent", "opaque", "auto"] """ - Allows to set transparency for the background of the generated image(s). This - parameter is only supported for GPT image models that support transparent - backgrounds. Must be one of `transparent`, `opaque`, or `auto` (default value). - When `auto` is used, the model will automatically determine the best background - for the image. - - `gpt-image-2` and `gpt-image-2-2026-04-21` do not support transparent - backgrounds. Requests with `background` set to `transparent` will return an - error for these models; use `opaque` or `auto` instead. - - If `transparent`, the output format needs to support transparency, so it should - be set to either `png` (default value) or `webp`. + Allows to set transparency for the background of the generated image(s). Must be + one of `transparent`, `opaque`, or `auto` (default value). When `auto` is used, + the model will automatically determine the best background for the image. + + Transparent backgrounds are available for supported GPT Image models. For + `gpt-image-2` and `gpt-image-2-2026-04-21`, this support is in preview. When + using `transparent`, set the output format to `png` or `webp`. """ input_fidelity: Optional[Literal["high", "low"]] From 04d5d79fc71e883a3114b80ed8408bef76e72cd3 Mon Sep 17 00:00:00 2001 From: Alex Chang Date: Fri, 21 Aug 2026 14:55:57 +0000 Subject: [PATCH 2/5] chore: set a ceiling for Python SDK customization (#3714) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Set a generous 10,000-line ceiling for the Python SDK's remaining customization of generated files. The current verified patch is 6,558 lines (+6,010 / −548), so normal development has headroom while unusually large patches need an explicit decision. This PR intentionally changes only `.castiron-ratchet.json`. Keeping the policy separate means an SDK change cannot raise its own allowance. A human approving review is required; the automated technical review is not policy approval. Companion tooling PR #3715 adds enforcement using the budget already on `main`. Land this policy first. This seed alone does not enable a required check, and it does not change SDK behavior or CODEOWNERS. --- .castiron-ratchet.json | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .castiron-ratchet.json diff --git a/.castiron-ratchet.json b/.castiron-ratchet.json new file mode 100644 index 0000000000..a5ee9ba654 --- /dev/null +++ b/.castiron-ratchet.json @@ -0,0 +1,4 @@ +{ + "schema_version": 1, + "max_custom_patch_lines": 10000 +} From 5daacfb6ff912445c23f9f9c9b3ab8394ef4e6c5 Mon Sep 17 00:00:00 2001 From: Alex Chang Date: Fri, 21 Aug 2026 15:24:02 +0000 Subject: [PATCH 3/5] ci: catch accidental growth in Python SDK custom patches (#3715) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary A small SDK diff can hide a large patch against generated output. Add a custom-code gate that counts custom additions and deletions together, using the budget and checker already on `main`. Extend the existing two custom-code workflows and reuse their trusted report—no new workflows or compiler changes. Keep the report comment available when the gate fails, preserve default CODEOWNERS, and require a separate justified, human-reviewed PR for budget increases. Budget-only #3714 is merged; this PR is rebased onto `main`. Enable `Castiron / budget-only change` and `Castiron / custom-code budget` only after the tooling is on `main` and the rollout checks in `scripts/castiron/CUSTOM_CODE.md` pass, retaining the required merge queue. `AGENTS.md` and `CONTRIBUTING.md` contain the agent rules and contributor policy and link to those technical details. This PR does not activate repository rules or enable auto-merge. --- .../castiron-custom-code-comment.yml | 111 ++- .github/workflows/castiron-custom-code.yml | 20 +- AGENTS.md | 10 + CONTRIBUTING.md | 17 + scripts/castiron/CUSTOM_CODE.md | 117 ++++ scripts/castiron/custom_code_budget.py | 430 ++++++++++++ scripts/castiron/test_custom_code_budget.py | 639 ++++++++++++++++++ 7 files changed, 1337 insertions(+), 7 deletions(-) create mode 100644 scripts/castiron/CUSTOM_CODE.md create mode 100644 scripts/castiron/custom_code_budget.py create mode 100644 scripts/castiron/test_custom_code_budget.py diff --git a/.github/workflows/castiron-custom-code-comment.yml b/.github/workflows/castiron-custom-code-comment.yml index dabe1b3817..e7218bcae0 100644 --- a/.github/workflows/castiron-custom-code-comment.yml +++ b/.github/workflows/castiron-custom-code-comment.yml @@ -15,7 +15,7 @@ concurrency: jobs: compute: name: Compute trusted custom-code report - if: github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.path == '.github/workflows/castiron-custom-code.yml' + if: contains(fromJSON('["pull_request", "merge_group"]'), github.event.workflow_run.event) && github.event.workflow_run.path == '.github/workflows/castiron-custom-code.yml' runs-on: ubuntu-latest timeout-minutes: 10 permissions: @@ -26,15 +26,20 @@ jobs: number: ${{ steps.report.outputs.number }} artifact-id: ${{ steps.artifact.outputs.artifact-id }} artifact-run-attempt: ${{ github.run_attempt }} + base-sha: ${{ steps.budget.outputs.base_sha }} + head-sha: ${{ steps.budget.outputs.head_sha }} + isolation: ${{ steps.budget.outputs.isolation }} + budget: ${{ steps.budget.outputs.budget }} steps: - name: Check out the trusted reporter uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: - ref: ${{ github.workflow_sha }} + ref: main persist-credentials: false - name: Compute from the current pull request Git objects id: report + if: github.event.workflow_run.event == 'pull_request' env: GH_TOKEN: ${{ github.token }} GIT_CONFIG_COUNT: '2' @@ -66,10 +71,110 @@ jobs: if-no-files-found: error retention-days: 7 + # The prior step's report was computed here from + # Git objects by main's reporter, not downloaded from the candidate run. + - name: Evaluate main's custom-code budget + id: budget + if: always() + continue-on-error: true # A budget failure must not suppress the existing report comment. + env: + GH_TOKEN: ${{ github.token }} + REPOSITORY: ${{ github.repository }} + SOURCE_EVENT: ${{ github.event.workflow_run.event }} + GIT_CONFIG_COUNT: '2' + GIT_CONFIG_KEY_0: credential.helper + GIT_CONFIG_VALUE_0: '' + GIT_CONFIG_KEY_1: credential.https://github.com.helper + GIT_CONFIG_VALUE_1: '!gh auth git-credential' + run: | + trusted_sha=$(git rev-parse HEAD) + reuse=() + if [[ "$SOURCE_EVENT" == pull_request ]]; then + reuse=(--trusted-report-dir "$RUNNER_TEMP/castiron-custom-code") + fi + python3 -I scripts/castiron/custom_code_budget.py github \ + --repository "$REPOSITORY" --event-path "$GITHUB_EVENT_PATH" \ + --trusted-sha "$trusted_sha" --repo "$RUNNER_TEMP/castiron-objects.git" \ + "${reuse[@]}" --out "$RUNNER_TEMP/custom-code-budget" + + - name: Add the budget to the run summary + if: always() + run: | + if test -f "$RUNNER_TEMP/custom-code-budget/summary.md"; then + cat "$RUNNER_TEMP/custom-code-budget/summary.md" >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Upload trusted budget measurements + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: custom-code-budget-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/custom-code-budget/ + if-no-files-found: error + retention-days: 7 + + budget-status: + name: Publish custom-code budget checks + needs: compute + if: always() && !cancelled() && needs.compute.result != 'skipped' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + actions: read + pull-requests: read + statuses: write + steps: + # This workflow definition is from main. No candidate checkout/artifacts. + - name: Publish exact-head statuses after checking freshness + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + env: + BASE_SHA: ${{ needs.compute.outputs.base-sha }} + HEAD_SHA: ${{ needs.compute.outputs.head-sha }} + ISOLATION_RESULT: ${{ needs.compute.outputs.isolation }} + BUDGET_RESULT: ${{ needs.compute.outputs.budget }} + with: + script: | + const event = context.payload.workflow_run; + const {data: run} = await github.rest.actions.getWorkflowRun({...context.repo, run_id: event.id}); + if (run.head_sha !== event.head_sha || run.run_attempt !== event.run_attempt || + run.status !== 'completed' || run.path.split('@', 1)[0] !== '.github/workflows/castiron-custom-code.yml' || + run.repository.full_name !== `${context.repo.owner}/${context.repo.repo}`) return; + const head = run.head_sha; + if (!/^[0-9a-f]{40}$/.test(head)) throw new Error('Invalid candidate SHA'); + const {data: main} = await github.rest.git.getRef({...context.repo, ref: 'heads/main'}); + const base = main.object.sha; + if (run.event === 'pull_request') { + const pulls = run.pull_requests.length ? run.pull_requests : await github.paginate( + github.rest.repos.listPullRequestsAssociatedWithCommit, {...context.repo, commit_sha: head}); + const current = []; + for (const pull of pulls) { + const {data: pr} = await github.rest.pulls.get({...context.repo, pull_number: pull.number}); + if (pr.state === 'open' && pr.head.sha === head && pr.base.sha === base && + pr.base.ref === 'main' && pr.base.repo.full_name === `${context.repo.owner}/${context.repo.repo}`) current.push(pr); + } + if (current.length !== 1) return; + } else if (run.event !== 'merge_group' || !run.head_branch.startsWith('gh-readonly-queue/main/')) { + return; + } + const fresh = base === process.env.BASE_SHA && head === process.env.HEAD_SHA; + const url = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + for (const [name, result] of [ + ['Castiron / budget-only change', process.env.ISOLATION_RESULT], + ['Castiron / custom-code budget', process.env.BUDGET_RESULT], + ]) { + const state = fresh && result === 'success' ? 'success' : 'failure'; + const description = !fresh ? 'Evaluation unavailable or base changed; rerun against current main.' + : state === 'success' ? 'Passed against main policy. See the trusted run summary.' + : 'Budget check failed. See the trusted run summary.'; + await github.rest.repos.createCommitStatus({...context.repo, sha: head, context: name, + state, description, target_url: url}); + } + comment: name: Update custom-code comment needs: compute - if: always() && !cancelled() && (needs.compute.result == 'failure' || needs.compute.outputs.number != '') + if: always() && !cancelled() && github.event.workflow_run.event == 'pull_request' && (needs.compute.result == 'failure' || needs.compute.outputs.number != '') runs-on: ubuntu-latest timeout-minutes: 5 permissions: diff --git a/.github/workflows/castiron-custom-code.yml b/.github/workflows/castiron-custom-code.yml index ef23bfc8b1..51bd5149aa 100644 --- a/.github/workflows/castiron-custom-code.yml +++ b/.github/workflows/castiron-custom-code.yml @@ -3,20 +3,32 @@ name: Castiron custom code on: pull_request: - types: [opened, synchronize, reopened, ready_for_review] + types: [opened, synchronize, reopened, ready_for_review, edited] + # Notify the existing trusted handler for queue candidates. + merge_group: + types: [checks_requested] permissions: contents: read concurrency: - group: castiron-custom-code-${{ github.event.pull_request.number }} + group: castiron-custom-code-${{ github.event.pull_request.number || github.event.merge_group.head_ref }} cancel-in-progress: false env: REPORTER_SHA256: ac48ca88e9f7ad57195038157e99f055c0cd3dac8de856e4d102dca807766d4a jobs: + queue-signal: + if: github.event_name == 'merge_group' + runs-on: ubuntu-latest + timeout-minutes: 1 + permissions: {} + steps: + - run: echo 'The workflow_run handler on main evaluates this candidate independently.' + report: + if: github.event_name == 'pull_request' name: Castiron / baseline consistency runs-on: ubuntu-latest timeout-minutes: 10 @@ -44,8 +56,8 @@ jobs: - name: Verify the reporter matches its generated workflow run: printf '%s %s\n' "$REPORTER_SHA256" scripts/castiron/custom_code_report.py | sha256sum --check --strict - - name: Test hash mismatch and snapshot isolation - run: python3 -m unittest discover -s scripts/castiron -p test_custom_code_report.py + - name: Test snapshot isolation and the custom-code budget + run: python3 -m unittest discover -s scripts/castiron -p 'test_custom_code*.py' - name: Validate the codegen hash and report custom code env: diff --git a/AGENTS.md b/AGENTS.md index c6547bd2d0..5bcdb9fb61 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,6 +7,16 @@ changing generated files. Handwritten policy, automation, tests, and examples should remain small and should not alter exported SDK APIs unless the change explicitly requires it. +## Custom-code budget + +Follow [the custom-code guidance](scripts/castiron/CUSTOM_CODE.md). Budget changes +belong in a separate PR containing only `.castiron-ratchet.json`, with an explicit justification +in the PR description. Increases require a **human approving review** before merging. +Agents may investigate and draft proposals, but must not approve budget increases +(including through a human's credentials) or bypass the gate. Do not weaken +counting, broaden exclusions, or alter generation metadata to make a change pass. +The checker and effective budget come from main, not the PR. Keep default CODEOWNERS. + ## Security requirements for coding agents - Never commit real API or admin keys, bearer tokens, webhook secrets, cloud diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9e215de1ac..b3cf767549 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -81,6 +81,23 @@ Most of the SDK is generated code. Modifications to code will be persisted betwe result in merge conflicts between manual patches and changes from the generator. The generator will never modify the contents of the `src/openai/lib/` and `examples/` directories. +## Custom-code budget + +The custom-code budget counts additions plus deletions in the remaining patch +against verified generated output. `.castiron-ratchet.json` defines this repository's +ceiling. CI uses the checker and budget on main, not the PR's proposed versions. + +Budget changes must be in a separate PR modifying **only `.castiron-ratchet.json`**. +Justify the current usage, proposed ceiling, and why fixing generation is not +appropriate in the PR description. Increases require a **human approving review** +and must merge before an SDK change relies on them. Agents may draft proposals, +but must not approve increases or bypass the gate. Keep default CODEOWNERS. +Lower the ceiling after cleanup while retaining headroom; decreases must still +fit the measured usage. + +See [custom-code technical details](scripts/castiron/CUSTOM_CODE.md) for accounting, +local checks, trusted CI, and activation instructions. + ## Security requirements for contributions - Never commit API or admin keys, bearer tokens, webhook secrets, cloud diff --git a/scripts/castiron/CUSTOM_CODE.md b/scripts/castiron/CUSTOM_CODE.md new file mode 100644 index 0000000000..2e819baf58 --- /dev/null +++ b/scripts/castiron/CUSTOM_CODE.md @@ -0,0 +1,117 @@ +# Custom code + +The custom-code reporter measures the SDK's remaining customization of generated +files. The budget gate checks that measurement against the repository's policy. +See [CONTRIBUTING.md](../../CONTRIBUTING.md#custom-code-budget) for budget changes +and human review, and [AGENTS.md](../../AGENTS.md#custom-code-budget) for agent rules. + +## What is counted + +The budget is **added lines plus deleted lines in the entire current custom patch** +against the candidate's verified pure-generated snapshot. It is not the PR's diff +against main, and additions and deletions never cancel. Replacing 500 generated +lines with 500 handwritten lines costs 1,000 lines. Deleting a generated file +counts all its removed lines. Restoring generated content reduces the budget. + +The checker reuses `scripts/castiron/custom_code_report.py` without changing its +snapshot/hash validation or generated-file ownership. It counts generated-owned +files, including runtime code, tests, and generated documentation. Wholly +handwritten-only files are outside this existing report's scope. Changes in +generation ownership remain visible in the report; do not change exclusions or +checkpoints to hide customization. Non-text custom patches fail as uncountable. + +The ceiling is defined in [`.castiron-ratchet.json`](../../.castiron-ratchet.json). +The policy file contains only an integer `schema_version` (currently 1) and a +nonnegative integer `max_custom_patch_lines`. Missing, deleted, renamed, executable, symlinked, +malformed, or unsupported policy files fail closed. There is no disable flag or +automatic budget update command. + +Budget-file isolation is checked against the complete PR diff, not just its latest +commit. Increases never apply to their own PR or merge group. Decreases must also +fit the measured usage. Human approval is contributor and agent policy plus normal +review, not an automated claim that GitHub can identify who operated an account. + +## Trusted CI + +The custom-code workflow pair separates candidate execution from trusted checks: + +- `castiron-custom-code.yml` runs proposed offline tests and the advisory report + on `pull_request` with read-only permissions. +- `castiron-custom-code-comment.yml` handles `workflow_run` from **main**. Its + read-only compute job runs main's reporter against candidate Git objects in a + new bare repository, then reuses that verified report to check main's budget. + It never checks out, imports, installs, or executes candidate code. +- An unprivileged `merge_group` job in the first workflow only signals that a candidate + needs checking. A `workflow_run` handler whose **workflow definition is on + main** then uses the same trusted checkout. It ignores the signal's conclusion + and artifacts, verifies the run and queue membership through GitHub and commit + ancestry, checks isolation per constituent PR, and measures the complete merged + candidate against actual main. A budget PR queued in the same group cannot + grant the SDK PR a higher limit. + +A separate publisher with no checkout attaches these statuses to the exact PR +head or merge-group SHA, after rechecking head/base freshness: + +- `Castiron / budget-only change` +- `Castiron / custom-code budget` + +The policy is read from the current base commit, not the PR or its merge base. +Reporter changes in a PR cannot change the checker executing on that PR. Missing +snapshots, invalid hashes, unavailable queue membership, and policy errors fail +closed. If main moves during evaluation, rerun the workflow; reruns check out the +new main. PR-head statuses are feedback at a point in time: a main update alone +does not rerun them. The checker therefore fails unless main has an effective +**require merge queue** rule. The queue must recheck combined usage against current +main before merging; do not replace that protection with PR-head statuses alone. + +The queue signal cannot supply a passing result, and removing it leaves required +statuses missing. No write-capable job executes from the candidate's workflow +definition. See GitHub's [workflow_run trust model](https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows#workflow_run) +and [merge-queue behavior](https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/configuring-pull-request-merges/managing-a-merge-queue). + +The trusted run summary reports additions, deletions, total, mixed-file count, +headroom, largest patches, and exact policy/candidate/generated revisions. The +existing custom-code comment remains unchanged, including when the budget fails. +The trusted compute job reuses its own report, never the candidate's artifacts. +The checker, policy, and workflows are maintained in the SDK and preserved +through the normal three-way merge during generation. + +## Local verification + +Run from the SDK repository with Python 3.10+; no SDK dependencies are needed: + +```sh +python3 -m unittest discover -s scripts/castiron -p 'test_custom_code*.py' +``` + +Use a **trusted checkout's** script to inspect committed revisions: + +```sh +python3 -I scripts/castiron/custom_code_budget.py check \ + --repo /path/to/sdk --base FULL_MAIN_SHA --head FULL_CANDIDATE_SHA \ + --public --fetch --out /tmp/custom-code-budget +``` + +The inspected repo's `origin` must match its snapshot source. Omit `--public` +for the private SDK repository's checkpoints. Local checks are diagnostic; CI +independently binds main and the candidate to live GitHub metadata. + +## Activation order + +1. Merge the initial budget policy in its budget-only PR after human review. +2. Merge the tooling and guidance separately. Do not enable required + statuses before the policy and trusted checker exist on main. There is no + permanent bootstrap exemption in the checker. +3. Exercise a passing PR, over-budget PR, mixed budget/code PR, a fork PR, and a + merge-queue candidate. Policy increases must still use the old limit. +4. Have a human administrator retain/enable **require merge queue** and add both + exact status names above to the main ruleset, bound to GitHub Actions. Retain + normal review and CODEOWNERS settings. Exercise a main-policy decrease after + an unchanged PR head passed, and confirm its queued candidate uses the new limit. + The tests job is separate from these authoritative statuses. + +This is an accidental-change guardrail, not an adversarial approval boundary. +Workflow definitions and ruleset changes still have the normal repository review +and administrator controls; GitHub Actions status names alone are not a unique +workflow identity. Stronger organization-required workflow enforcement can be +added later. diff --git a/scripts/castiron/custom_code_budget.py b/scripts/castiron/custom_code_budget.py new file mode 100644 index 0000000000..e131d9a5a3 --- /dev/null +++ b/scripts/castiron/custom_code_budget.py @@ -0,0 +1,430 @@ +#!/usr/bin/env python3 +"""SDK custom-code budget gate. Run only from a trusted checkout, never PR code. + +Reuses Castiron's vendored snapshot verifier and generated-file accounting. This +file and its workflow are maintained in the SDK repository. +""" + +from __future__ import annotations + +import argparse +import html +import importlib.util +import json +import os +import subprocess +import sys +from pathlib import Path +from typing import Any, cast + +# Explicitly load the reporter beside this trusted script, including under python -I. +# The inspected repository is only a Git object store, not a Python import path. +_spec = importlib.util.spec_from_file_location( + "castiron_budget_reporter", Path(__file__).with_name("custom_code_report.py") +) +assert _spec is not None and _spec.loader is not None +report = importlib.util.module_from_spec(_spec) +sys.modules[_spec.name] = report +_spec.loader.exec_module(report) + +POLICY = ".castiron-ratchet.json" +ISOLATION_MESSAGE = ( + "Budget changes require a separate, budget-only PR. " + "Put the justification in the PR description." +) + + +def unique_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + value: dict[str, Any] = {} + for key, item in pairs: + if key in value: + raise ValueError(f"duplicate policy key: {key}") + value[key] = item + return value + + +def read_budget(repo: Path, revision: str) -> int: + revision = report.require_sha(revision) + entry = report.git(repo, "ls-tree", revision, "--", POLICY).split() + if len(entry) != 4 or entry[:2] != [b"100644", b"blob"]: + raise ValueError(f"{POLICY} must exist as a regular, non-executable file at {revision}") + if int(report.git(repo, "cat-file", "-s", entry[2].decode())) > 4096: + raise ValueError("budget file exceeds 4096 bytes") + value = json.loads( + report.git(repo, "show", f"{revision}:{POLICY}"), object_pairs_hook=unique_keys + ) + if not isinstance(value, dict): + raise ValueError("budget file must be an object") + value = cast(dict[str, Any], value) + if set(value) != {"schema_version", "max_custom_patch_lines"}: + raise ValueError("budget file must contain only schema_version and max_custom_patch_lines") + if type(value["schema_version"]) is not int or value["schema_version"] != 1: + raise ValueError("unsupported budget schema_version") + limit = value["max_custom_patch_lines"] + if type(limit) is not int or limit < 0: + raise ValueError("max_custom_patch_lines must be a nonnegative integer") + return limit + + +def changed_paths(repo: Path, base: str, head: str) -> set[bytes]: + """Use the entire PR diff, not its last commit or a truncated API file list.""" + start = ( + report.git(repo, "merge-base", report.require_sha(base), report.require_sha(head)) + .decode() + .strip() + ) + return set( + report.git( + repo, + "diff", + "--name-only", + "-z", + "--no-renames", + "--no-ext-diff", + "--no-textconv", + start, + head, + "--", + ).split(b"\0") + ) - {b""} + + +def check_isolation(repo: Path, base: str, head: str) -> int | None: + # Validate even if the policy was removed by an earlier commit in this PR. + proposed = read_budget(repo, head) + paths = changed_paths(repo, base, head) + if POLICY.encode() not in paths: + return None + if paths != {POLICY.encode()}: + raise ValueError(ISOLATION_MESSAGE) + return proposed + + +def count_custom_lines(result: dict[str, Any]) -> tuple[int, int]: + if result.get("status") != "ok": + raise ValueError("custom-code report could not verify the generated baseline") + added = removed = 0 + for file in result["files"]: + if not file["custom_after"]: + continue + counts = [file["added"], file["removed"]] + if any(not isinstance(n, str) or not n.isascii() or not n.isdecimal() for n in counts): + raise ValueError(f"cannot count non-text customization: {file['path']}") + added += int(counts[0]) + removed += int(counts[1]) + return added, removed + + +def outcome(state: str, description: str) -> dict[str, str]: + return {"state": state, "description": description} + + +def evaluate( + repo: Path, + base: str, + head: str, + *, + public: bool, + fetch: bool = False, + pull_heads: list[str] | None = None, + measurement: tuple[dict[str, Any], bytes] | None = None, +) -> tuple[dict[str, Any], bytes]: + """In a queue, isolate each PR but budget the complete merged candidate.""" + base, head = report.require_sha(base), report.require_sha(head) + result: dict[str, Any] = {"base_sha": base, "head_sha": head, "checks": {}} + checks = result["checks"] + proposed_limits: list[int] = [] + try: + read_budget(repo, base) + read_budget(repo, head) + if pull_heads is not None and not pull_heads: + raise ValueError("merge group has no verified constituent PRs") + for pr_head in pull_heads if pull_heads is not None else [head]: + proposed = check_isolation(repo, base, pr_head) + if proposed is not None: + proposed_limits.append(proposed) + checks["isolation"] = outcome( + "success", "Budget changes are isolated; human review is required for increases." + ) + except (report.ReportError, ValueError, UnicodeError) as exc: + checks["isolation"] = outcome("failure", str(exc)) + + patch = b"" + try: + limit = read_budget(repo, base) + result["limit"] = limit + measured, patch = ( + measurement + if measurement is not None + else report.build_report( + repo, + base, + head, + public=public, + fetch=fetch, + require_head_hash=True, + ) + ) + added, removed = count_custom_lines(measured) + total = added + removed + result.update( + additions=added, + deletions=removed, + total=total, + mixed_files=measured["counts"]["after"], + generated=measured["after"], + files=measured["files"], + ) + # Increases never apply to their own PR/group. Decreases must be viable too. + checked_limit = min([limit, *proposed_limits]) + result["checked_limit"] = checked_limit + state = "success" if total <= checked_limit else "failure" + checks["budget"] = outcome( + state, f"+{added} / -{removed} = {total} custom lines; limit {checked_limit}." + ) + except (report.ReportError, ValueError, UnicodeError, KeyError) as exc: + checks["budget"] = outcome("failure", str(exc)) + return result, patch + + +def write_result(out: Path, result: dict[str, Any], patch: bytes = b"") -> None: + out.mkdir(parents=True, exist_ok=True) + (out / "budget.json").write_text(json.dumps(result, indent=2, sort_keys=True) + "\n") + (out / "custom-code.patch").write_bytes(patch) + lines = ["## Custom-code budget", ""] + for name in ("isolation", "budget"): + check = result["checks"][name] + lines.append(f"- **{name}: {check['state']}** — {html.escape(check['description'])}") + if "total" in result: + lines += [ + "", + f"Mixed files: **{result['mixed_files']}**. Base-budget headroom: **{result['limit'] - result['total']} lines**.", + ] + lines += [ + "", + f"Policy: `{result['base_sha']}` · candidate: `{result['head_sha']}`.", + f"Verified generated snapshot: `{result['generated']['commit']}`.", + "", + "| Largest custom patches | Added | Deleted |", + "|---|---:|---:|", + ] + files = [f for f in result["files"] if f["custom_after"]] + for file in sorted(files, key=lambda f: int(f["added"]) + int(f["removed"]), reverse=True)[ + :10 + ]: + path = html.escape(file["path"]).replace("|", "|").replace("\n", " ") + lines.append(f"| {path} | {file['added']} | {file['removed']} |") + ownership = [f for f in result["files"] if f["category"] == "no_longer_generated"] + if ownership: + lines += ["", "**No longer generated (outside the current budget):**"] + lines += [f"- {html.escape(f['path'])}" for f in ownership] + lines += [ + "", + "Budget increases require a separate budget-only PR, explicit justification, and a human approving review. Agents may not approve or bypass an increase.", + ] + summary = "\n".join(lines) + "\n" + (out / "summary.md").write_text(summary) + print(summary) + + +def queued_entries(repository: str, branch: str) -> list[tuple[str, str]]: + """Query the actual queue; never infer PR membership from commit messages.""" + owner, name = repository.split("/") + query = """query($owner:String!,$name:String!,$branch:String!,$cursor:String) { + repository(owner:$owner,name:$name) { mergeQueue(branch:$branch) { + entries(first:100,after:$cursor) { + nodes { pullRequest { headRefOid } headCommit { oid } } + pageInfo { hasNextPage endCursor } + } + } } + }""" + cursor = None + heads: list[tuple[str, str]] = [] + while True: + response = report.api( + "POST", + "graphql", + { + "query": query, + "variables": {"owner": owner, "name": name, "branch": branch, "cursor": cursor}, + }, + ) + entries = response["data"]["repository"]["mergeQueue"]["entries"] + for node in entries["nodes"]: + # Entries still awaiting preparation have no synthetic queue commit. + if node["headCommit"] is not None: + heads.append( + ( + report.require_sha(node["pullRequest"]["headRefOid"]), + report.require_sha(node["headCommit"]["oid"]), + ) + ) + page = entries["pageInfo"] + if not page["hasNextPage"]: + return heads + if not page["endCursor"] or page["endCursor"] == cursor: + raise ValueError("invalid merge-queue pagination") + cursor = page["endCursor"] + + +def github_evaluate( + repo: Path, + repository: str, + event: dict[str, Any], + trusted_sha: str, + trusted_report_dir: Path | None = None, +) -> tuple[dict[str, Any], bytes]: + """Fetch data into a fresh bare repo; the executable checkout remains trusted.""" + if not report.REPOSITORY.fullmatch(repository): + raise ValueError("invalid repository") + root = f"repos/{repository}" + metadata = report.api("GET", root) + if event["repository"]["full_name"] != repository: + raise ValueError("event repository mismatch") + branch = metadata["default_branch"] + main = report.require_sha(report.api("GET", f"{root}/git/ref/heads/{branch}")["object"]["sha"]) + if report.require_sha(trusted_sha) != main: + raise ValueError("trusted checkout is stale; rerun against current main") + signal = event["workflow_run"] + run_id = signal["id"] + if type(run_id) is not int or run_id <= 0: + raise ValueError("invalid source run ID") + run = report.api("GET", f"{root}/actions/runs/{run_id}") + if ( + run["head_sha"] != signal["head_sha"] + or run["repository"]["full_name"] != repository + or run["path"].split("@", 1)[0] != ".github/workflows/castiron-custom-code.yml" + or run["status"] != "completed" + or run["run_attempt"] != signal["run_attempt"] + ): + raise ValueError("unexpected or superseded source workflow run") + head = report.require_sha(run["head_sha"]) + if run["event"] == "pull_request": + associated = run["pull_requests"] or report.api( + "GET", f"{root}/commits/{head}/pulls?per_page=100" + ) + current: list[int] = [] + for number in sorted({int(pr["number"]) for pr in associated}): + if number <= 0: + raise ValueError("invalid associated PR number") + pull = report.api("GET", f"{root}/pulls/{number}") + if ( + pull["state"] == "open" + and pull["head"]["sha"] == head + and pull["base"]["repo"]["full_name"] == repository + and pull["base"]["ref"] == branch + and pull["base"]["sha"] == main + ): + current.append(number) + if len(current) != 1: + raise ValueError("source run must identify exactly one current PR targeting main") + elif run["event"] == "merge_group": + if not run["head_branch"].startswith(f"gh-readonly-queue/{branch}/"): + raise ValueError("queue signal does not target main") + else: + raise ValueError("unsupported budget event") + # PR statuses can outlive their base revision. Require the queue's fresh, + # combined candidate check before merging rather than relying on those alone. + rules = report.api("GET", f"{root}/rules/branches/{branch}") + if not any(rule["type"] == "merge_queue" for rule in rules): + raise ValueError("main must require a merge queue before activating the budget gate") + measurement = None + if trusted_report_dir is not None: + # Only the preceding main-branch step may supply this directory. Never + # download a candidate artifact here. Reuse its verified report and bare + # object store, avoiding a second snapshot fetch/hash/diff computation. + if run["event"] != "pull_request": + raise ValueError("only PR runs can reuse the trusted report") + measured = json.loads((trusted_report_dir / "report.json").read_text()) + if measured["target_base_sha"] != main or measured["head_sha"] != head: + raise ValueError("trusted report is stale; rerun against current main") + if report.git(repo, "rev-parse", "--is-bare-repository").strip() != b"true": + raise ValueError("trusted report must use a bare object store") + measurement = (measured, (trusted_report_dir / "custom-code.patch").read_bytes()) + else: + if repo.exists(): + raise ValueError("Git object directory must be fresh") + subprocess.run(["git", "init", "--bare", str(repo)], check=True, capture_output=True) + report.git(repo, "remote", "add", "origin", f"https://github.com/{repository}.git") + report.git(repo, "fetch", "--quiet", "--no-tags", "origin", main, head) + pull_heads: list[str] | None = None + if run["event"] == "merge_group": + if report.git(repo, "merge-base", main, head).decode().strip() != main: + raise ValueError("merge group does not include current main; retry the current group") + pull_heads = [] + entries = queued_entries(repository, branch) + if not any(queue_head == head for _, queue_head in entries): + raise ValueError("merge-group head is no longer in the current queue") + for candidate, queue_head in entries: + report.git(repo, "fetch", "--quiet", "--no-tags", "origin", queue_head) + ancestor = report.git(repo, "merge-base", queue_head, head).decode().strip() + if ancestor == queue_head: + report.git(repo, "fetch", "--quiet", "--no-tags", "origin", candidate) + pull_heads.append(candidate) + if not pull_heads: + raise ValueError("cannot verify merge-group PR membership; retry the current group") + return evaluate( + repo, + main, + head, + public=not metadata["private"], + fetch=True, + pull_heads=pull_heads, + measurement=measurement, + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + local = commands.add_parser("check") + local.add_argument("--base", required=True) + local.add_argument("--head", required=True) + local.add_argument("--public", action="store_true") + local.add_argument("--fetch", action="store_true") + github = commands.add_parser("github") + github.add_argument("--repository", required=True) + github.add_argument("--event-path", type=Path, required=True) + github.add_argument("--trusted-sha", required=True) + github.add_argument("--trusted-report-dir", type=Path) + for command in (local, github): + command.add_argument("--repo", type=Path, required=True) + command.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + try: + if args.command == "check": + result, patch = evaluate( + args.repo, args.base, args.head, public=args.public, fetch=args.fetch + ) + else: + result, patch = github_evaluate( + args.repo, + args.repository, + json.loads(args.event_path.read_text()), + args.trusted_sha, + args.trusted_report_dir, + ) + except ( + report.ReportError, + ValueError, + OSError, + KeyError, + TypeError, + subprocess.SubprocessError, + ) as exc: + result = { + "checks": {name: outcome("failure", str(exc)) for name in ("isolation", "budget")} + } + patch = b"" + write_result(args.out, result, patch) + if args.command == "github" and "GITHUB_OUTPUT" in os.environ: + with Path(os.environ["GITHUB_OUTPUT"]).open("a") as output: + for name in ("isolation", "budget"): + output.write(f"{name}={result['checks'][name]['state']}\n") + output.write(f"base_sha={result.get('base_sha', '')}\n") + output.write(f"head_sha={result.get('head_sha', '')}\n") + return 0 if all(c["state"] == "success" for c in result["checks"].values()) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/castiron/test_custom_code_budget.py b/scripts/castiron/test_custom_code_budget.py new file mode 100644 index 0000000000..a2d01b8c26 --- /dev/null +++ b/scripts/castiron/test_custom_code_budget.py @@ -0,0 +1,639 @@ +# Regression tests for the custom-code budget. +from __future__ import annotations + +import json +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from typing import Any, cast +from unittest import mock + +import custom_code_budget as budget +import test_custom_code_report as fixtures + + +def source_run(head: str, event: str = "pull_request") -> dict[str, Any]: + return { + "id": 123, + "event": event, + "head_sha": head, + "head_branch": "gh-readonly-queue/main/pr-7-example" if event == "merge_group" else "sdk", + "repository": {"full_name": "openai/example"}, + "path": ".github/workflows/castiron-custom-code.yml", + "status": "completed", + "run_attempt": 1, + "pull_requests": [{"number": 3}] if event == "pull_request" else [], + "conclusion": "failure", # The candidate's result is deliberately ignored. + } + + +class BudgetTests(unittest.TestCase): + def setUp(self) -> None: # pyright: ignore[reportImplicitOverride] + # Reuse the reporter's real Git/checkpoint fixture without inheriting its tests. + self.fixture = fixtures.CustomCodeTests() + self.fixture.setUp() + self.addCleanup(self.fixture.doCleanups) + self.repo = self.fixture.repo + self.generated, _ = self.fixture.baseline() + self.policy(10) + self.base = self.fixture.commit("human-owned budget") + + def policy(self, limit: int) -> None: + self.fixture.write( + budget.POLICY, json.dumps({"schema_version": 1, "max_custom_patch_lines": limit}) + "\n" + ) + + def evaluate(self, head: str, base: str | None = None, **kwargs: Any) -> dict[str, Any]: + return budget.evaluate(self.repo, base or self.base, head, public=False, **kwargs)[0] + + def test_additions_and_deletions_do_not_cancel(self) -> None: + self.fixture.write("generated.py", "replacement\n") + result = self.evaluate(self.fixture.commit()) + self.assertEqual((result["additions"], result["deletions"], result["total"]), (1, 1, 2)) + self.assertEqual(result["checks"]["budget"]["state"], "success") + + def test_below_equal_and_above_limit(self) -> None: + for additions in (9, 10, 11): + with self.subTest(additions=additions): + self.fixture.write("generated.py", "generated\n" + "custom\n" * additions) + result = self.evaluate(self.fixture.commit()) + self.assertEqual(result["total"], additions) + self.assertEqual( + result["checks"]["budget"]["state"], "failure" if additions > 10 else "success" + ) + + def test_whole_generated_file_deletion_is_counted(self) -> None: + (self.repo / "generated.py").unlink() + result = self.evaluate(self.fixture.commit()) + self.assertEqual((result["additions"], result["deletions"]), (0, 1)) + self.assertEqual(result["mixed_files"], 1) + + def test_restoring_generated_content_removes_customization(self) -> None: + self.fixture.write("generated.py", "generated\ncustom\n") + customized = self.fixture.commit() + self.fixture.write("generated.py", "generated\n") + result = self.evaluate(self.fixture.commit(), base=customized) + self.assertEqual(result["total"], 0) + self.assertEqual(result["mixed_files"], 0) + + def test_handwritten_only_files_keep_existing_report_scope(self) -> None: + self.fixture.write("handwritten.py", "custom\n" * 100) + self.assertEqual(self.evaluate(self.fixture.commit())["total"], 0) + + def test_increase_is_isolated_but_does_not_apply_to_itself(self) -> None: + self.policy(20) + result = self.evaluate(self.fixture.commit()) + self.assertEqual(result["checks"]["isolation"]["state"], "success") + self.assertEqual(result["limit"], 10) + self.assertEqual(result["checked_limit"], 10) + + def test_entire_pr_must_be_budget_only_not_just_latest_commit(self) -> None: + self.fixture.write("generated.py", "generated\n" + "custom\n" * 11) + self.fixture.commit("SDK change first") + self.policy(100) + result = self.evaluate(self.fixture.commit("budget-only last commit")) + self.assertEqual(result["checks"]["isolation"]["state"], "failure") + self.assertIn("separate, budget-only PR", result["checks"]["isolation"]["description"]) + self.assertEqual(result["checks"]["budget"]["state"], "failure") + self.assertEqual(result["limit"], 10) + + def test_new_base_budget_is_used_for_stale_pr_branch(self) -> None: + self.fixture.git("checkout", "-q", "-b", "sdk", self.base) + self.fixture.write("generated.py", "generated\n" + "custom\n" * 11) + head = self.fixture.commit() + self.fixture.git("checkout", "-q", "main") + self.policy(20) + new_base = self.fixture.commit("separate approved increase") + result = self.evaluate(head, base=new_base) + self.assertEqual(result["limit"], 20) + self.assertEqual(result["checks"]["isolation"]["state"], "success") + self.assertEqual(result["checks"]["budget"]["state"], "success") + + def test_decrease_must_fit_current_usage(self) -> None: + self.fixture.write("generated.py", "generated\ncustom\ncustom\n") + base = self.fixture.commit() + self.policy(1) + result = self.evaluate(self.fixture.commit(), base=base) + self.assertEqual(result["checks"]["isolation"]["state"], "success") + self.assertEqual(result["checks"]["budget"]["state"], "failure") + self.assertEqual(result["checked_limit"], 1) + + def test_missing_base_policy_fails_closed(self) -> None: + (self.repo / budget.POLICY).unlink() + base = self.fixture.commit() + self.policy(100) + result = self.evaluate(self.fixture.commit(), base=base) + self.assertTrue(all(c["state"] == "failure" for c in result["checks"].values())) + + def test_policy_deletion_rename_symlink_and_mode_change_fail(self) -> None: + for change in ("delete", "rename", "symlink", "executable"): + with self.subTest(change=change): + self.fixture.git("checkout", "--detach", "-q", self.base) + path = self.repo / budget.POLICY + if change == "delete": + path.unlink() + elif change == "rename": + path.rename(self.repo / "renamed.json") + elif change == "symlink": + path.unlink() + path.symlink_to("generated.py") + else: + path.chmod(0o755) + result = self.evaluate(self.fixture.commit()) + self.assertEqual(result["checks"]["isolation"]["state"], "failure") + + def test_invalid_policy_values_fail(self) -> None: + invalid = [ + "[]", + "{}", + "not json", + '{"schema_version":1,"max_custom_patch_lines":true}', + '{"schema_version":true,"max_custom_patch_lines":10}', + '{"schema_version":2,"max_custom_patch_lines":10}', + '{"schema_version":1,"max_custom_patch_lines":-1}', + '{"schema_version":1,"max_custom_patch_lines":10.5}', + '{"schema_version":1,"max_custom_patch_lines":null}', + '{"schema_version":1,"max_custom_patch_lines":10,"mode":"report-only"}', + '{"schema_version":1,"max_custom_patch_lines":10,"max_custom_patch_lines":999}', + " " * 4097, + ] + for contents in invalid: + with self.subTest(contents=contents[:100]): + self.fixture.write(budget.POLICY, contents) + result = self.evaluate(self.fixture.commit()) + self.assertEqual(result["checks"]["isolation"]["state"], "failure") + + def test_bad_snapshot_and_binary_change_fail_budget(self) -> None: + self.fixture.write("generated.py", "binary\0content") + result = self.evaluate(self.fixture.commit()) + self.assertIn("non-text", result["checks"]["budget"]["description"]) + self.fixture.write("generated.py", "generated\n") + path = self.repo / ".castiron.stats.yml" + path.write_text( + path.read_text().replace( + budget.report.hash_codegen_commit(self.repo, self.generated), "f" * 64 + ) + ) + result = self.evaluate(self.fixture.commit()) + self.assertIn("codegen_hash mismatch", result["checks"]["budget"]["description"]) + + def test_budget_uses_existing_reporter_with_strict_verification(self) -> None: + with mock.patch.object( + budget.report, "build_report", wraps=budget.report.build_report + ) as measured: + self.evaluate(self.base) + self.assertEqual( + measured.call_args.kwargs, {"public": False, "fetch": False, "require_head_hash": True} + ) + + def test_queue_isolates_prs_but_uses_main_budget_for_combined_tree(self) -> None: + self.fixture.git("checkout", "-q", "-b", "policy", self.base) + self.policy(100) + policy_head = self.fixture.commit() + self.fixture.git("checkout", "-q", "-b", "sdk", self.base) + self.fixture.write("generated.py", "generated\n" + "custom\n" * 11) + sdk_head = self.fixture.commit() + self.fixture.git("checkout", "-q", "-b", "queue", self.base) + self.fixture.git("merge", "--no-ff", "-m", "queue policy", policy_head) + self.fixture.git("merge", "--no-ff", "-m", "queue SDK", sdk_head) + result = self.evaluate( + self.fixture.git("rev-parse", "HEAD"), pull_heads=[policy_head, sdk_head] + ) + self.assertEqual(result["checks"]["isolation"]["state"], "success") + self.assertEqual(result["checks"]["budget"]["state"], "failure") + self.assertEqual(result["limit"], 10) + + def test_queue_without_verified_members_fails(self) -> None: + self.assertEqual( + self.evaluate(self.base, pull_heads=[])["checks"]["isolation"]["state"], "failure" + ) + + def test_queue_membership_uses_synthetic_commit_not_original_pr_ancestry(self) -> None: + self.fixture.git("checkout", "-q", "-b", "sdk", self.base) + self.fixture.write("generated.py", "generated\ncustom\n") + pr_head = self.fixture.commit() + self.fixture.git("checkout", "-q", "-b", "queue", self.base) + self.fixture.git("merge", "--squash", "sdk") + queue_head = self.fixture.commit("synthetic queue commit") + self.assertNotEqual(self.fixture.git("merge-base", pr_head, queue_head), pr_head) + event = { + "repository": {"full_name": "openai/example"}, + "workflow_run": source_run(queue_head, "merge_group"), + } + original_git = budget.report.git + + def local_fetch(repo: Path, *args: str) -> bytes: + if args[0] == "fetch": + args = tuple(str(self.repo) if arg == "origin" else arg for arg in args) + return cast(bytes, original_git(repo, *args)) + + with ( + tempfile.TemporaryDirectory() as temp, + mock.patch.object( + budget.report, + "api", + side_effect=[ + {"default_branch": "main", "private": False}, + {"object": {"sha": self.base}}, + source_run(queue_head, "merge_group"), + [{"type": "merge_queue"}], + ], + ), + mock.patch.object(budget, "queued_entries", return_value=[(pr_head, queue_head)]), + mock.patch.object( + budget.report, + "git", + side_effect=local_fetch, + ), + mock.patch.object(budget, "evaluate", return_value=({}, b"")) as evaluate, + ): + repo = Path(temp) / "queue.git" + budget.github_evaluate(repo, "openai/example", event, self.base) + evaluate.assert_called_once_with( + repo, + self.base, + queue_head, + public=True, + fetch=True, + pull_heads=[pr_head], + measurement=None, + ) + + def test_cli_executes_trusted_reporter_not_inspected_repo(self) -> None: + self.fixture.write( + "scripts/castiron/custom_code_report.py", 'raise RuntimeError("PR CODE EXECUTED")\n' + ) + self.fixture.write("sitecustomize.py", 'raise RuntimeError("PR IMPORTED")\n') + head = self.fixture.commit() + out = self.repo / "result" + command = [ + sys.executable, + "-I", + str(Path(budget.__file__).resolve()), + "check", + "--repo", + str(self.repo), + "--base", + self.base, + "--head", + head, + "--out", + str(out), + ] + completed = subprocess.run(command, cwd=self.repo, capture_output=True, text=True) + self.assertEqual(completed.returncode, 0, completed.stderr + completed.stdout) + self.assertEqual(json.loads((out / "budget.json").read_text())["total"], 0) + + def test_summary_has_counts_revisions_and_no_longer_generated(self) -> None: + self.fixture.write("generated.py", "replacement\n") + result = self.evaluate(self.fixture.commit()) + out = self.repo / "report-output" + budget.write_result(out, result) + summary = (out / "summary.md").read_text() + self.assertIn("+1 / -1 = 2", summary) + self.assertIn(self.generated, summary) + self.assertIn(self.base, summary) + self.assertIn("human approving review", summary) + + +@unittest.skipUnless(shutil.which("node"), "Node is needed to execute the status-publisher fixture") +class StatusPublisherTests(unittest.TestCase): + def publish( + self, + *, + event_name: str = "pull_request", + head_changed: bool = False, + base_changed: bool = False, + no_result: bool = False, + failed_budget: bool = False, + run_overrides: dict[str, Any] | None = None, + ) -> list[dict[str, Any]]: + path = ( + Path(__file__).resolve().parents[2] + / ".github/workflows/castiron-custom-code-comment.yml" + ) + section = path.read_text().split("\n budget-status:\n", 1)[1].split("\n comment:\n", 1)[0] + publisher = section.split(" script: |\n", 1)[1] + script = "\n".join(line[12:] for line in publisher.splitlines()) + base, head = "a" * 40, "b" * 40 + payload = { + "script": script, + "context": { + "eventName": "workflow_run", + "repo": {"owner": "openai", "repo": "example"}, + "serverUrl": "https://github.com", + "runId": 123, + "payload": { + "workflow_run": source_run(head, event_name), + }, + }, + "run": {**source_run(head, event_name), **(run_overrides or {})}, + "current": { + "state": "open", + "head": {"sha": "c" * 40 if head_changed else head}, + "base": { + "sha": "c" * 40 if base_changed else base, + "ref": "main", + "repo": {"full_name": "openai/example"}, + }, + }, + "env": { + "BASE_SHA": "" if no_result else base, + "HEAD_SHA": "" if no_result else head, + "ISOLATION_RESULT": "success", + "BUDGET_RESULT": "failure" if failed_budget else "success", + }, + } + harness = """ + const fs = require('node:fs'); + const data = JSON.parse(fs.readFileSync(0, 'utf8')); + const published = []; + const github = {rest: { + pulls: {get: async () => ({data: data.current})}, + actions: {getWorkflowRun: async () => ({data: data.run})}, + git: {getRef: async () => ({data: {object: {sha: data.current.base.sha}}})}, + repos: {createCommitStatus: async value => published.push(value)}, + }}; + const AsyncFunction = Object.getPrototypeOf(async function(){}).constructor; + new AsyncFunction('github','context','process', data.script)(github, data.context, {env:data.env}) + .then(() => process.stdout.write(JSON.stringify(published))) + .catch(error => { console.error(error); process.exitCode = 1; }); + """ + output = subprocess.run( + ["node", "-e", harness], + input=json.dumps(payload), + text=True, + capture_output=True, + check=True, + ) + return cast(list[dict[str, Any]], json.loads(output.stdout)) + + def test_statuses_attach_to_candidate_not_main(self) -> None: + for event in ("pull_request", "merge_group"): + with self.subTest(event=event): + results = self.publish(event_name=event) + self.assertEqual(len(results), 2) + self.assertTrue( + all(r["sha"] == "b" * 40 and r["state"] == "success" for r in results) + ) + + def test_stale_pr_head_is_not_published(self) -> None: + self.assertEqual(self.publish(head_changed=True), []) + + def test_stale_base_and_missing_evaluation_cannot_publish_success(self) -> None: + for event in ("pull_request", "merge_group"): + for base_changed, no_result in ((True, False), (False, True)): + results = self.publish( + event_name=event, base_changed=base_changed, no_result=no_result + ) + self.assertEqual(len(results), 2) + self.assertTrue(all(r["state"] == "failure" for r in results)) + + def test_superseded_or_wrong_source_run_cannot_publish(self) -> None: + for overrides in ( + {"run_attempt": 2}, + {"head_sha": "c" * 40}, + {"event": "push"}, + {"path": "other.yml"}, + ): + with self.subTest(overrides=overrides): + self.assertEqual(self.publish(run_overrides=overrides), []) + + def test_independent_check_failures_are_preserved(self) -> None: + results = self.publish(failed_budget=True) + self.assertEqual([r["state"] for r in results], ["success", "failure"]) + + +class GitHubBudgetTests(unittest.TestCase): + def test_merge_queue_rule_is_required_before_passing(self) -> None: + base, head = "a" * 40, "b" * 40 + event = { + "repository": {"full_name": "openai/example"}, + "workflow_run": source_run(head, "merge_group"), + } + with ( + tempfile.TemporaryDirectory() as temp, + mock.patch.object( + budget.report, + "api", + side_effect=[ + {"default_branch": "main"}, + {"object": {"sha": base}}, + source_run(head, "merge_group"), + [{"type": "required_status_checks"}], + ], + ), + ): + repo = Path(temp) / "objects.git" + with self.assertRaisesRegex(ValueError, "must require a merge queue"): + budget.github_evaluate(repo, "openai/example", event, base) + self.assertFalse(repo.exists()) + + def test_wrong_or_superseded_source_runs_fail_before_fetching(self) -> None: + base, head = "a" * 40, "b" * 40 + event = {"repository": {"full_name": "openai/example"}, "workflow_run": source_run(head)} + for overrides in ( + {"run_attempt": 2}, + {"status": "in_progress"}, + {"path": "other.yml"}, + {"head_sha": "c" * 40}, + {"repository": {"full_name": "wrong/repo"}}, + ): + with ( + self.subTest(overrides=overrides), + tempfile.TemporaryDirectory() as temp, + mock.patch.object( + budget.report, + "api", + side_effect=[ + {"default_branch": "main"}, + {"object": {"sha": base}}, + {**source_run(head), **overrides}, + ], + ), + ): + repo = Path(temp) / "objects.git" + with self.assertRaisesRegex(ValueError, "source workflow run"): + budget.github_evaluate(repo, "openai/example", event, base) + self.assertFalse(repo.exists()) + + def test_reuses_only_matching_main_job_report(self) -> None: + base, head = "a" * 40, "b" * 40 + event = {"repository": {"full_name": "openai/example"}, "workflow_run": source_run(head)} + pull = { + "state": "open", + "head": {"sha": head}, + "base": { + "sha": base, + "ref": "main", + "repo": {"full_name": "openai/example"}, + }, + } + for stale in (False, True): + with self.subTest(stale=stale), tempfile.TemporaryDirectory() as temp: + trusted = Path(temp) + measured = {"target_base_sha": "c" * 40 if stale else base, "head_sha": head} + (trusted / "report.json").write_text(json.dumps(measured)) + (trusted / "custom-code.patch").write_bytes(b"verified patch") + repo = trusted / "objects.git" + repo.mkdir() + with ( + mock.patch.object( + budget.report, + "api", + side_effect=[ + {"default_branch": "main", "private": False}, + {"object": {"sha": base}}, + source_run(head), + pull, + [{"type": "merge_queue"}], + ], + ), + mock.patch.object(budget.report, "git", return_value=b"true\n") as git, + mock.patch.object(budget, "evaluate", return_value=({}, b"")) as evaluate, + ): + if stale: + with self.assertRaisesRegex(ValueError, "trusted report is stale"): + budget.github_evaluate(repo, "openai/example", event, base, trusted) + evaluate.assert_not_called() + else: + budget.github_evaluate(repo, "openai/example", event, base, trusted) + evaluate.assert_called_once_with( + repo, + base, + head, + public=True, + fetch=True, + pull_heads=None, + measurement=(measured, b"verified patch"), + ) + git.assert_called_once_with(repo, "rev-parse", "--is-bare-repository") + + def test_queue_pagination(self) -> None: + pages = [ + { + "nodes": [ + {"pullRequest": {"headRefOid": "a" * 40}, "headCommit": {"oid": "c" * 40}} + ], + "pageInfo": {"hasNextPage": True, "endCursor": "cursor"}, + }, + { + "nodes": [ + {"pullRequest": {"headRefOid": "b" * 40}, "headCommit": {"oid": "d" * 40}} + ], + "pageInfo": {"hasNextPage": False, "endCursor": None}, + }, + ] + with mock.patch.object( + budget.report, + "api", + side_effect=[{"data": {"repository": {"mergeQueue": {"entries": p}}}} for p in pages], + ) as api: + self.assertEqual( + budget.queued_entries("openai/example", "main"), + [("a" * 40, "c" * 40), ("b" * 40, "d" * 40)], + ) + self.assertEqual(api.call_args_list[1].args[2]["variables"]["cursor"], "cursor") + + def test_pull_context_refuses_stale_checkout_before_fetching(self) -> None: + with ( + tempfile.TemporaryDirectory() as temp, + mock.patch.object( + budget.report, + "api", + side_effect=[{"default_branch": "main"}, {"object": {"sha": "a" * 40}}], + ), + ): + repo = Path(temp) / "objects.git" + with self.assertRaisesRegex(ValueError, "trusted checkout is stale"): + budget.github_evaluate( + repo, + "openai/example", + {"repository": {"full_name": "openai/example"}}, + "b" * 40, + ) + self.assertFalse(repo.exists()) + + def test_github_uses_fresh_bare_data_repo_and_checks_current_head(self) -> None: + base, head = "a" * 40, "b" * 40 + event = { + "repository": {"full_name": "openai/example"}, + "workflow_run": source_run(head), + } + pull = { + "state": "open", + "head": {"sha": head}, + "base": {"sha": base, "ref": "main", "repo": {"full_name": "openai/example"}}, + } + responses = [ + {"default_branch": "main", "private": False}, + {"object": {"sha": base}}, + source_run(head), + pull, + [{"type": "merge_queue"}], + ] + with ( + tempfile.TemporaryDirectory() as temp, + mock.patch.object(budget.report, "api", side_effect=responses), + mock.patch.object(budget.report, "git") as git, + mock.patch.object(budget, "evaluate", return_value=({}, b"")) as evaluate, + ): + repo = Path(temp) / "objects.git" + budget.github_evaluate(repo, "openai/example", event, base) + self.assertTrue((repo / "HEAD").exists()) + self.assertFalse((repo / "src").exists()) + self.assertIn( + mock.call(repo, "fetch", "--quiet", "--no-tags", "origin", base, head), + git.call_args_list, + ) + evaluate.assert_called_once_with( + repo, base, head, public=True, fetch=True, pull_heads=None, measurement=None + ) + + def test_stale_pull_and_wrong_target_fail_before_objects_created(self) -> None: + for kind in ("head", "base", "repository", "branch", "closed"): + with self.subTest(kind=kind), tempfile.TemporaryDirectory() as temp: + base, head = "a" * 40, "b" * 40 + event = { + "repository": {"full_name": "openai/example"}, + "workflow_run": source_run(head), + } + pull: dict[str, Any] = { + "state": "open", + "head": {"sha": head}, + "base": {"sha": base, "ref": "main", "repo": {"full_name": "openai/example"}}, + } + if kind == "head": + pull["head"]["sha"] = "c" * 40 + elif kind == "base": + pull["base"]["sha"] = "c" * 40 + elif kind == "repository": + pull["base"]["repo"]["full_name"] = "wrong/repo" + elif kind == "branch": + pull["base"]["ref"] = "other" + else: + pull["state"] = "closed" + with mock.patch.object( + budget.report, + "api", + side_effect=[ + {"default_branch": "main"}, + {"object": {"sha": base}}, + source_run(head), + pull, + ], + ): + with self.assertRaisesRegex(ValueError, "exactly one current PR"): + budget.github_evaluate( + Path(temp) / "objects.git", + "openai/example", + event, + base, + ) + + +if __name__ == "__main__": + unittest.main() From a002ef3c7d2276fb63deeea58d920d87da61b1e5 Mon Sep 17 00:00:00 2001 From: Alex Chang Date: Fri, 21 Aug 2026 15:26:46 +0000 Subject: [PATCH 4/5] refactor(api): isolate vector-store polling helpers (#3713) - [x] I understand that this repository is auto-generated and my pull request may not be merged ## Changes being requested Move the existing sync/async vector-store file and file-batch polling bodies into SDK-owned `lib/_vector_stores.py`, sharing their identical `openai-poll-after-ms` lookup, integer conversion, and one-second fallback. The public `poll` methods remain thin delegates with the same signatures, defaults, docstrings, and return types. Request headers, response parsing, interval handling, terminal conditions, return identity, and error/cancellation propagation are preserved. Upload orchestration, create-and-poll methods, other requests, and response wrappers are untouched. This adds no generic polling framework, timeout behavior, schema/compiler changes, or generation-metadata changes. Review pointers: - Original public polling bodies: [Files](https://github.com/openai/openai-python/blob/bedb9a7b8839e193107e88b92f7cc166f08ac83d/src/openai/resources/vector_stores/files.py#L360) and [FileBatches](https://github.com/openai/openai-python/blob/bedb9a7b8839e193107e88b92f7cc166f08ac83d/src/openai/resources/vector_stores/file_batches.py#L331), with the corresponding async methods in the same files. - [SDK-owned helper](https://github.com/openai/openai-python/blob/25e95b31c1402b3a927cedb9fff5d3d7a785bbeb/src/openai/lib/_vector_stores.py#L16): the four bodies only rename their receiver and replace the repeated interval block with the shared lookup. Source/AST comparison against the base verifies those edits and the unchanged remainder of both resources. - [Focused tests](https://github.com/openai/openai-python/blob/25e95b31c1402b3a927cedb9fff5d3d7a785bbeb/tests/lib/test_vector_store_polling.py#L47): 106 lines / 22 cases covering the four public entry points, terminal results, default/server/explicit intervals and headers, retrieval errors, and async cancellation. Existing API and helper-signature tests are unchanged. ## Additional context & links Validation: - Commands: `.venv/bin/python -m pytest tests/lib/test_vector_store_polling.py -q -n 0` and the same command with `.venv-pydantic-v1/bin/python` passed all **22 cases against the original implementation and again after extraction**. - Commands: `TEST_API_BASE_URL=http://127.0.0.1:4142 .venv/bin/python -m pytest tests/api_resources/vector_stores tests/lib/test_vector_store_files.py tests/lib/test_vector_store_file_batches.py tests/lib/test_vector_store_polling.py -q -n 0` and the same command with `.venv-pydantic-v1/bin/python` passed **248 tests in each Pydantic mode**. - Commands: `./scripts/format` and `./scripts/lint` passed, including Ruff, Pyright, mypy, and import checks. Unrelated reporter-formatting edits were excluded. - Command: `./scripts/build` passed. Both distributions include the helper, and a fresh import from the built wheel loads the helper and all four resource classes. The verified custom-code report keeps **36 mixed files**, with only the two vector-store resource customizations changed: - `files.py`: **+222/-4 to +179/-1**. - `file_batches.py`: **+275/-4 to +242/-3**. All other 34 customizations, generation metadata, API reference, dependencies, and workflows are unchanged. Reproduction command: ```sh python3 scripts/castiron/custom_code_report.py report \ --base bedb9a7b8839e193107e88b92f7cc166f08ac83d \ --head 25e95b31c1402b3a927cedb9fff5d3d7a785bbeb \ --fetch --require-head-hash --public \ --out /tmp/castiron-vector-polling ``` --- src/openai/lib/_vector_stores.py | 149 ++++++++++++++++++ .../resources/vector_stores/file_batches.py | 66 ++------ src/openai/resources/vector_stores/files.py | 78 +++------ tests/lib/test_vector_store_polling.py | 106 +++++++++++++ 4 files changed, 291 insertions(+), 108 deletions(-) create mode 100644 src/openai/lib/_vector_stores.py create mode 100644 tests/lib/test_vector_store_polling.py diff --git a/src/openai/lib/_vector_stores.py b/src/openai/lib/_vector_stores.py new file mode 100644 index 0000000000..7357a4908e --- /dev/null +++ b/src/openai/lib/_vector_stores.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Mapping +from typing_extensions import assert_never + +from .._types import Omit +from .._utils import is_given +from ..types.vector_stores.vector_store_file import VectorStoreFile +from ..types.vector_stores.vector_store_file_batch import VectorStoreFileBatch + +if TYPE_CHECKING: + from ..resources.vector_stores.files import Files, AsyncFiles + from ..resources.vector_stores.file_batches import FileBatches, AsyncFileBatches + + +def _get_poll_interval_ms(headers: Mapping[str, str]) -> int: + """Use the server hint or the existing one-second default.""" + from_header = headers.get("openai-poll-after-ms") + if from_header is not None: + return int(from_header) + return 1000 + + +def poll_vector_store_file( + resource: Files, + file_id: str, + *, + vector_store_id: str, + poll_interval_ms: int | Omit, +) -> VectorStoreFile: + """Poll a vector-store file using the caller's resource hooks.""" + headers: dict[str, str] = {"X-Stainless-Poll-Helper": "true"} + if is_given(poll_interval_ms): + headers["X-Stainless-Custom-Poll-Interval"] = str(poll_interval_ms) + + while True: + response = resource.with_raw_response.retrieve( + file_id, + vector_store_id=vector_store_id, + extra_headers=headers, + ) + + file = response.parse() + if file.status == "in_progress": + if not is_given(poll_interval_ms): + poll_interval_ms = _get_poll_interval_ms(response.headers) + + resource._sleep(poll_interval_ms / 1000) + elif file.status == "cancelled" or file.status == "completed" or file.status == "failed": + return file + else: + if TYPE_CHECKING: # type: ignore[unreachable] + assert_never(file.status) + else: + return file + + +async def async_poll_vector_store_file( + resource: AsyncFiles, + file_id: str, + *, + vector_store_id: str, + poll_interval_ms: int | Omit, +) -> VectorStoreFile: + """Poll a vector-store file using the caller's async resource hooks.""" + headers: dict[str, str] = {"X-Stainless-Poll-Helper": "true"} + if is_given(poll_interval_ms): + headers["X-Stainless-Custom-Poll-Interval"] = str(poll_interval_ms) + + while True: + response = await resource.with_raw_response.retrieve( + file_id, + vector_store_id=vector_store_id, + extra_headers=headers, + ) + + file = response.parse() + if file.status == "in_progress": + if not is_given(poll_interval_ms): + poll_interval_ms = _get_poll_interval_ms(response.headers) + + await resource._sleep(poll_interval_ms / 1000) + elif file.status == "cancelled" or file.status == "completed" or file.status == "failed": + return file + else: + if TYPE_CHECKING: # type: ignore[unreachable] + assert_never(file.status) + else: + return file + + +def poll_vector_store_file_batch( + resource: FileBatches, + batch_id: str, + *, + vector_store_id: str, + poll_interval_ms: int | Omit, +) -> VectorStoreFileBatch: + """Poll a vector-store batch using the caller's resource hooks.""" + headers: dict[str, str] = {"X-Stainless-Poll-Helper": "true"} + if is_given(poll_interval_ms): + headers["X-Stainless-Custom-Poll-Interval"] = str(poll_interval_ms) + + while True: + response = resource.with_raw_response.retrieve( + batch_id, + vector_store_id=vector_store_id, + extra_headers=headers, + ) + + batch = response.parse() + if batch.file_counts.in_progress > 0: + if not is_given(poll_interval_ms): + poll_interval_ms = _get_poll_interval_ms(response.headers) + + resource._sleep(poll_interval_ms / 1000) + continue + + return batch + + +async def async_poll_vector_store_file_batch( + resource: AsyncFileBatches, + batch_id: str, + *, + vector_store_id: str, + poll_interval_ms: int | Omit, +) -> VectorStoreFileBatch: + """Poll a vector-store batch using the caller's async resource hooks.""" + headers: dict[str, str] = {"X-Stainless-Poll-Helper": "true"} + if is_given(poll_interval_ms): + headers["X-Stainless-Custom-Poll-Interval"] = str(poll_interval_ms) + + while True: + response = await resource.with_raw_response.retrieve( + batch_id, + vector_store_id=vector_store_id, + extra_headers=headers, + ) + + batch = response.parse() + if batch.file_counts.in_progress > 0: + if not is_given(poll_interval_ms): + poll_interval_ms = _get_poll_interval_ms(response.headers) + + await resource._sleep(poll_interval_ms / 1000) + continue + + return batch diff --git a/src/openai/resources/vector_stores/file_batches.py b/src/openai/resources/vector_stores/file_batches.py index 9bd5040172..853ba3a565 100644 --- a/src/openai/resources/vector_stores/file_batches.py +++ b/src/openai/resources/vector_stores/file_batches.py @@ -13,13 +13,17 @@ from ... import _legacy_response from ...types import FileChunkingStrategyParam from ..._types import Body, Omit, Query, Headers, NotGiven, FileTypes, SequenceNotStr, omit, not_given -from ..._utils import is_given, path_template, maybe_transform, async_maybe_transform +from ..._utils import path_template, maybe_transform, async_maybe_transform from ..._compat import cached_property from ..._resource import SyncAPIResource, AsyncAPIResource from ..._response import to_streamed_response_wrapper, async_to_streamed_response_wrapper from ...pagination import SyncCursorPage, AsyncCursorPage from ..._base_client import AsyncPaginator, make_request_options from ...types.file_object import FileObject +from ...lib._vector_stores import ( + poll_vector_store_file_batch as _poll_vector_store_file_batch, + async_poll_vector_store_file_batch as _async_poll_vector_store_file_batch, +) from ...types.vector_stores import file_batch_create_params, file_batch_list_files_params from ...types.file_chunking_strategy_param import FileChunkingStrategyParam from ...types.vector_stores.vector_store_file import VectorStoreFile @@ -340,30 +344,12 @@ def poll( Note: this will return even if one of the files failed to process, you need to check batch.file_counts.failed_count to handle this case. """ - headers: dict[str, str] = {"X-Stainless-Poll-Helper": "true"} - if is_given(poll_interval_ms): - headers["X-Stainless-Custom-Poll-Interval"] = str(poll_interval_ms) - - while True: - response = self.with_raw_response.retrieve( - batch_id, - vector_store_id=vector_store_id, - extra_headers=headers, - ) - - batch = response.parse() - if batch.file_counts.in_progress > 0: - if not is_given(poll_interval_ms): - from_header = response.headers.get("openai-poll-after-ms") - if from_header is not None: - poll_interval_ms = int(from_header) - else: - poll_interval_ms = 1000 - - self._sleep(poll_interval_ms / 1000) - continue - - return batch + return _poll_vector_store_file_batch( + self, + batch_id, + vector_store_id=vector_store_id, + poll_interval_ms=poll_interval_ms, + ) def upload_and_poll( self, @@ -728,30 +714,12 @@ async def poll( Note: this will return even if one of the files failed to process, you need to check batch.file_counts.failed_count to handle this case. """ - headers: dict[str, str] = {"X-Stainless-Poll-Helper": "true"} - if is_given(poll_interval_ms): - headers["X-Stainless-Custom-Poll-Interval"] = str(poll_interval_ms) - - while True: - response = await self.with_raw_response.retrieve( - batch_id, - vector_store_id=vector_store_id, - extra_headers=headers, - ) - - batch = response.parse() - if batch.file_counts.in_progress > 0: - if not is_given(poll_interval_ms): - from_header = response.headers.get("openai-poll-after-ms") - if from_header is not None: - poll_interval_ms = int(from_header) - else: - poll_interval_ms = 1000 - - await self._sleep(poll_interval_ms / 1000) - continue - - return batch + return await _async_poll_vector_store_file_batch( + self, + batch_id, + vector_store_id=vector_store_id, + poll_interval_ms=poll_interval_ms, + ) async def upload_and_poll( self, diff --git a/src/openai/resources/vector_stores/files.py b/src/openai/resources/vector_stores/files.py index 06ae07129d..02b6ef2143 100644 --- a/src/openai/resources/vector_stores/files.py +++ b/src/openai/resources/vector_stores/files.py @@ -2,20 +2,24 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Dict, Union, Optional -from typing_extensions import Literal, assert_never +from typing import Dict, Union, Optional +from typing_extensions import Literal import httpx2 from ... import _legacy_response from ...types import FileChunkingStrategyParam from ..._types import Body, Omit, Query, Headers, NotGiven, FileTypes, omit, not_given -from ..._utils import is_given, path_template, maybe_transform, async_maybe_transform +from ..._utils import path_template, maybe_transform, async_maybe_transform from ..._compat import cached_property from ..._resource import SyncAPIResource, AsyncAPIResource from ..._response import to_streamed_response_wrapper, async_to_streamed_response_wrapper from ...pagination import SyncPage, AsyncPage, SyncCursorPage, AsyncCursorPage from ..._base_client import AsyncPaginator, make_request_options +from ...lib._vector_stores import ( + poll_vector_store_file as _poll_vector_store_file, + async_poll_vector_store_file as _async_poll_vector_store_file, +) from ...types.vector_stores import file_list_params, file_create_params, file_update_params from ...types.file_chunking_strategy_param import FileChunkingStrategyParam from ...types.vector_stores.vector_store_file import VectorStoreFile @@ -369,34 +373,12 @@ def poll( Note: this will return even if the file failed to process, you need to check file.last_error and file.status to handle these cases """ - headers: dict[str, str] = {"X-Stainless-Poll-Helper": "true"} - if is_given(poll_interval_ms): - headers["X-Stainless-Custom-Poll-Interval"] = str(poll_interval_ms) - - while True: - response = self.with_raw_response.retrieve( - file_id, - vector_store_id=vector_store_id, - extra_headers=headers, - ) - - file = response.parse() - if file.status == "in_progress": - if not is_given(poll_interval_ms): - from_header = response.headers.get("openai-poll-after-ms") - if from_header is not None: - poll_interval_ms = int(from_header) - else: - poll_interval_ms = 1000 - - self._sleep(poll_interval_ms / 1000) - elif file.status == "cancelled" or file.status == "completed" or file.status == "failed": - return file - else: - if TYPE_CHECKING: # type: ignore[unreachable] - assert_never(file.status) - else: - return file + return _poll_vector_store_file( + self, + file_id, + vector_store_id=vector_store_id, + poll_interval_ms=poll_interval_ms, + ) def upload( self, @@ -823,34 +805,12 @@ async def poll( Note: this will return even if the file failed to process, you need to check file.last_error and file.status to handle these cases """ - headers: dict[str, str] = {"X-Stainless-Poll-Helper": "true"} - if is_given(poll_interval_ms): - headers["X-Stainless-Custom-Poll-Interval"] = str(poll_interval_ms) - - while True: - response = await self.with_raw_response.retrieve( - file_id, - vector_store_id=vector_store_id, - extra_headers=headers, - ) - - file = response.parse() - if file.status == "in_progress": - if not is_given(poll_interval_ms): - from_header = response.headers.get("openai-poll-after-ms") - if from_header is not None: - poll_interval_ms = int(from_header) - else: - poll_interval_ms = 1000 - - await self._sleep(poll_interval_ms / 1000) - elif file.status == "cancelled" or file.status == "completed" or file.status == "failed": - return file - else: - if TYPE_CHECKING: # type: ignore[unreachable] - assert_never(file.status) - else: - return file + return await _async_poll_vector_store_file( + self, + file_id, + vector_store_id=vector_store_id, + poll_interval_ms=poll_interval_ms, + ) async def upload( self, diff --git a/tests/lib/test_vector_store_polling.py b/tests/lib/test_vector_store_polling.py new file mode 100644 index 0000000000..1e1c1d62e1 --- /dev/null +++ b/tests/lib/test_vector_store_polling.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +import asyncio +from typing import Any, cast +from unittest import mock +from typing_extensions import TypeAlias + +import pytest + +from openai._types import Omit, omit +from openai._models import construct_type_unchecked +from openai.resources.vector_stores.files import Files, AsyncFiles +from openai.resources.vector_stores.file_batches import FileBatches, AsyncFileBatches +from openai.types.vector_stores.vector_store_file import VectorStoreFile +from openai.types.vector_stores.vector_store_file_batch import VectorStoreFileBatch + +PollResource: TypeAlias = Files | AsyncFiles | FileBatches | AsyncFileBatches +ITEM_ID = "item-synthetic" +VECTOR_STORE_ID = "vs-synthetic" + + +@pytest.fixture(params=[Files, AsyncFiles, FileBatches, AsyncFileBatches]) +def resource(request: pytest.FixtureRequest) -> PollResource: + resource_type = cast(type[PollResource], request.param) + return resource_type(cast(Any, mock.Mock())) + + +def raw_response(resource: PollResource, *, pending: bool = False, headers: dict[str, str] | None = None) -> mock.Mock: + result: VectorStoreFile | VectorStoreFileBatch + if isinstance(resource, (Files, AsyncFiles)): + result = construct_type_unchecked( + type_=VectorStoreFile, value={"id": ITEM_ID, "status": "in_progress" if pending else "completed"} + ) + else: + result = construct_type_unchecked( + type_=VectorStoreFileBatch, value={"id": ITEM_ID, "file_counts": {"in_progress": int(pending)}} + ) + return mock.Mock(headers=headers or {}, parse=mock.Mock(return_value=result)) + + +async def poll(resource: PollResource, interval: int | Omit = omit) -> VectorStoreFile | VectorStoreFileBatch: + if isinstance(resource, (AsyncFiles, AsyncFileBatches)): + return await resource.poll(ITEM_ID, vector_store_id=VECTOR_STORE_ID, poll_interval_ms=interval) + return resource.poll(ITEM_ID, vector_store_id=VECTOR_STORE_ID, poll_interval_ms=interval) + + +async def test_terminal_result(resource: PollResource) -> None: + terminal = raw_response(resource) + with ( + mock.patch.object(resource.with_raw_response, "retrieve", return_value=terminal) as retrieve, + mock.patch.object(resource, "_sleep") as sleep, + ): + assert await poll(resource) is terminal.parse.return_value + retrieve.assert_called_once_with( + ITEM_ID, vector_store_id=VECTOR_STORE_ID, extra_headers={"X-Stainless-Poll-Helper": "true"} + ) + sleep.assert_not_called() + + +@pytest.mark.parametrize( + ("interval", "headers", "seconds"), + [(omit, {}, 1.0), (omit, {"openai-poll-after-ms": "2000"}, 2.0), (250, {"openai-poll-after-ms": "2000"}, 0.25)], + ids=["default", "server-hint", "explicit"], +) +async def test_poll_interval_and_headers( + resource: PollResource, interval: int | Omit, headers: dict[str, str], seconds: float +) -> None: + terminal = raw_response(resource) + with ( + mock.patch.object( + resource.with_raw_response, + "retrieve", + side_effect=[raw_response(resource, pending=True, headers=headers), terminal], + ) as retrieve, + mock.patch.object(resource, "_sleep") as sleep, + ): + assert await poll(resource, interval) is terminal.parse.return_value + expected_headers = {"X-Stainless-Poll-Helper": "true"} + if isinstance(interval, int): + expected_headers["X-Stainless-Custom-Poll-Interval"] = str(interval) + assert ( + retrieve.call_args_list + == [mock.call(ITEM_ID, vector_store_id=VECTOR_STORE_ID, extra_headers=expected_headers)] * 2 + ) + sleep.assert_called_once_with(seconds) + + +async def test_retrieve_error_propagates(resource: PollResource) -> None: + error = RuntimeError("synthetic retrieval failure") + with mock.patch.object(resource.with_raw_response, "retrieve", side_effect=error): + with pytest.raises(RuntimeError) as caught: + await poll(resource) + assert caught.value is error + + +@pytest.mark.parametrize("resource_type", [AsyncFiles, AsyncFileBatches]) +async def test_async_cancellation_propagates(resource_type: type[AsyncFiles] | type[AsyncFileBatches]) -> None: + resource = resource_type(cast(Any, mock.Mock())) + cancellation = asyncio.CancelledError() + with ( + mock.patch.object(resource.with_raw_response, "retrieve", return_value=raw_response(resource, pending=True)), + mock.patch.object(resource, "_sleep", side_effect=cancellation), + ): + with pytest.raises(asyncio.CancelledError) as caught: + await poll(resource) + assert caught.value is cancellation From 5f20c51c694801cb0a4d0fed413bedc1bac2dda0 Mon Sep 17 00:00:00 2001 From: Alex Chang Date: Fri, 21 Aug 2026 15:36:10 +0000 Subject: [PATCH 5/5] refactor(api): move file processing polling into SDK-owned helpers (#3712) - [x] I understand that this repository is auto-generated and my pull request may not be merged ## Changes being requested Move the existing sync and async file-processing polling bodies into SDK-owned `lib/_files.py`. `Files.wait_for_processing` and `AsyncFiles.wait_for_processing` keep their public signatures, defaults, docstrings, and return types, and delegate to the helpers. No schema, compiler, dependency, or generation-metadata changes are needed. The polling bodies are unchanged apart from renaming the receiver from `self` to `files`. This preserves the `processed`/`error`/`deleted` terminal states, wall-clock timing, dynamic `retrieve` and `_sleep` hooks, return identity, and exact errors. In particular, the strict `>` timeout check still happens after a repeated retrieval, even if that retrieval returns a terminal state. This does not introduce a generic polling framework or change existing timeout behavior. Review pointers: - Original public methods: [sync](https://github.com/openai/openai-python/blob/bedb9a7b8839e193107e88b92f7cc166f08ac83d/src/openai/resources/files.py#L366) and [async](https://github.com/openai/openai-python/blob/bedb9a7b8839e193107e88b92f7cc166f08ac83d/src/openai/resources/files.py#L720). - [Extracted bodies](https://github.com/openai/openai-python/blob/3dfdf85e4d33a3274270aadccda6278c41ac8ede/src/openai/lib/_files.py#L12). - [Focused regression tests](https://github.com/openai/openai-python/blob/3dfdf85e4d33a3274270aadccda6278c41ac8ede/tests/lib/test_file_processing.py#L36): 13 cases covering sync/async terminal results, polling, timeout and retrieval-error propagation, plus async cancellation. Following the [review discussion](https://github.com/openai/openai-python/pull/3712#discussion_r3830698221), the new handwritten suite is limited to 87 lines rather than an exhaustive matrix of clock, timeout, and event-order details. Existing Files API tests are unchanged. The review revision changes only this new test file; runtime source remains byte-identical to the original PR head. ## Additional context & links Validation: - Commands: `TEST_API_BASE_URL=http://127.0.0.1:4141 .venv/bin/python -m pytest tests/lib/test_file_processing.py tests/api_resources/test_files.py -q -n 0` and the same command using `.venv-pydantic-v1/bin/python`: **130 passed, 3 skipped** in each Pydantic mode. The skips are the existing aiohttp/respx2-incompatible `test_method_content`, `test_raw_response_content`, and `test_streaming_response_content` cases. - Commands: `.venv/bin/ruff format tests/lib/test_file_processing.py` and `./scripts/lint` passed on the review revision, including Ruff, Pyright, mypy, and import checks. The initial full `./scripts/format` also passed; unrelated reporter-formatting changes were excluded. - Command: `./scripts/build` passed. Both the wheel and source distribution contain `openai/lib/_files.py`; importing the helper and resource classes from the built wheel also passed. - Exact source/AST comparison against the base verified both polling bodies, all other resource statements, public signatures, exports, request construction, and response wrappers. The verified custom-code report keeps 36 mixed files and changes only the Files resource customization, shrinking it from **+47/-0 to +26/-0**. The other 35 customizations are unchanged. Reproduction command: ```sh python3 scripts/castiron/custom_code_report.py report \ --base bedb9a7b8839e193107e88b92f7cc166f08ac83d \ --head 3dfdf85e4d33a3274270aadccda6278c41ac8ede \ --fetch --require-head-hash --public \ --out /tmp/castiron-file-processing ``` --- src/openai/lib/_files.py | 57 ++++++++++++++++++++ src/openai/resources/files.py | 37 +++---------- tests/lib/test_file_processing.py | 87 +++++++++++++++++++++++++++++++ 3 files changed, 152 insertions(+), 29 deletions(-) create mode 100644 src/openai/lib/_files.py create mode 100644 tests/lib/test_file_processing.py diff --git a/src/openai/lib/_files.py b/src/openai/lib/_files.py new file mode 100644 index 0000000000..ce88c689f2 --- /dev/null +++ b/src/openai/lib/_files.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +import time +from typing import TYPE_CHECKING + +from ..types.file_object import FileObject + +if TYPE_CHECKING: + from ..resources.files import Files, AsyncFiles + + +def wait_for_file_processing( + files: Files, + id: str, + *, + poll_interval: float, + max_wait_seconds: float, +) -> FileObject: + """Poll a file using the caller's resource and sleep hooks.""" + TERMINAL_STATES = {"processed", "error", "deleted"} + + start = time.time() + file = files.retrieve(id) + while file.status not in TERMINAL_STATES: + files._sleep(poll_interval) + + file = files.retrieve(id) + if time.time() - start > max_wait_seconds: + raise RuntimeError( + f"Giving up on waiting for file {id} to finish processing after {max_wait_seconds} seconds." + ) + + return file + + +async def async_wait_for_file_processing( + files: AsyncFiles, + id: str, + *, + poll_interval: float, + max_wait_seconds: float, +) -> FileObject: + """Poll a file using the caller's async resource and sleep hooks.""" + TERMINAL_STATES = {"processed", "error", "deleted"} + + start = time.time() + file = await files.retrieve(id) + while file.status not in TERMINAL_STATES: + await files._sleep(poll_interval) + + file = await files.retrieve(id) + if time.time() - start > max_wait_seconds: + raise RuntimeError( + f"Giving up on waiting for file {id} to finish processing after {max_wait_seconds} seconds." + ) + + return file diff --git a/src/openai/resources/files.py b/src/openai/resources/files.py index aee4fd1ab2..820866685c 100644 --- a/src/openai/resources/files.py +++ b/src/openai/resources/files.py @@ -2,7 +2,6 @@ from __future__ import annotations -import time import typing_extensions from typing import Mapping, cast from typing_extensions import Literal @@ -24,6 +23,10 @@ to_custom_streamed_response_wrapper, async_to_custom_streamed_response_wrapper, ) +from ..lib._files import ( + wait_for_file_processing as _wait_for_file_processing, + async_wait_for_file_processing as _async_wait_for_file_processing, +) from ..pagination import SyncCursorPage, AsyncCursorPage from .._base_client import AsyncPaginator, make_request_options from ..types.file_object import FileObject @@ -371,20 +374,7 @@ def wait_for_processing( max_wait_seconds: float = 30 * 60, ) -> FileObject: """Waits for the given file to be processed, default timeout is 30 mins.""" - TERMINAL_STATES = {"processed", "error", "deleted"} - - start = time.time() - file = self.retrieve(id) - while file.status not in TERMINAL_STATES: - self._sleep(poll_interval) - - file = self.retrieve(id) - if time.time() - start > max_wait_seconds: - raise RuntimeError( - f"Giving up on waiting for file {id} to finish processing after {max_wait_seconds} seconds." - ) - - return file + return _wait_for_file_processing(self, id, poll_interval=poll_interval, max_wait_seconds=max_wait_seconds) class AsyncFiles(AsyncAPIResource): @@ -725,20 +715,9 @@ async def wait_for_processing( max_wait_seconds: float = 30 * 60, ) -> FileObject: """Waits for the given file to be processed, default timeout is 30 mins.""" - TERMINAL_STATES = {"processed", "error", "deleted"} - - start = time.time() - file = await self.retrieve(id) - while file.status not in TERMINAL_STATES: - await self._sleep(poll_interval) - - file = await self.retrieve(id) - if time.time() - start > max_wait_seconds: - raise RuntimeError( - f"Giving up on waiting for file {id} to finish processing after {max_wait_seconds} seconds." - ) - - return file + return await _async_wait_for_file_processing( + self, id, poll_interval=poll_interval, max_wait_seconds=max_wait_seconds + ) class FilesWithRawResponse: diff --git a/tests/lib/test_file_processing.py b/tests/lib/test_file_processing.py new file mode 100644 index 0000000000..8ac02acfcc --- /dev/null +++ b/tests/lib/test_file_processing.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import asyncio +from typing import Any, cast +from unittest import mock + +import pytest + +from openai import OpenAI, AsyncOpenAI +from openai.lib import _files as file_helpers +from openai._models import construct_type_unchecked +from openai.resources.files import Files, AsyncFiles +from openai.types.file_object import FileObject + +FILE_ID = "file-synthetic" + + +def make_file(status: str) -> FileObject: + return construct_type_unchecked(type_=FileObject, value={"id": FILE_ID, "status": status}) + + +@pytest.fixture(params=["sync", "async"]) +def files_resource(request: pytest.FixtureRequest) -> Files | AsyncFiles: + client = mock.Mock() + if request.param == "sync": + return Files(cast(OpenAI, client)) + return AsyncFiles(cast(AsyncOpenAI, client)) + + +async def wait(resource: Files | AsyncFiles, **kwargs: Any) -> FileObject: + if isinstance(resource, AsyncFiles): + return await resource.wait_for_processing(FILE_ID, **kwargs) + return resource.wait_for_processing(FILE_ID, **kwargs) + + +@pytest.mark.parametrize("status", ["processed", "error", "deleted"]) +async def test_terminal_result(files_resource: Files | AsyncFiles, status: str) -> None: + terminal = make_file(status) + with ( + mock.patch.object(files_resource, "retrieve", return_value=terminal) as retrieve, + mock.patch.object(files_resource, "_sleep") as sleep, + ): + assert await wait(files_resource) is terminal + retrieve.assert_called_once_with(FILE_ID) + sleep.assert_not_called() + + +async def test_poll_until_processed(files_resource: Files | AsyncFiles) -> None: + terminal = make_file("processed") + with ( + mock.patch.object(files_resource, "retrieve", side_effect=[make_file("uploaded"), terminal]) as retrieve, + mock.patch.object(files_resource, "_sleep") as sleep, + ): + assert await wait(files_resource, poll_interval=0.25) is terminal + assert retrieve.call_count == 2 + sleep.assert_called_once_with(0.25) + + +async def test_timeout(files_resource: Files | AsyncFiles) -> None: + with ( + mock.patch.object(file_helpers, "time") as clock, + mock.patch.object(files_resource, "retrieve", return_value=make_file("uploaded")), + mock.patch.object(files_resource, "_sleep"), + ): + clock.time.side_effect = [0.0, 11.0] + with pytest.raises(RuntimeError, match=f"Giving up on waiting for file {FILE_ID}"): + await wait(files_resource, max_wait_seconds=10) + + +async def test_retrieve_error_propagates(files_resource: Files | AsyncFiles) -> None: + error = RuntimeError("synthetic retrieval failure") + with mock.patch.object(files_resource, "retrieve", side_effect=error): + with pytest.raises(RuntimeError) as caught: + await wait(files_resource) + assert caught.value is error + + +async def test_async_cancellation_propagates() -> None: + resource = AsyncFiles(cast(AsyncOpenAI, mock.Mock())) + cancellation = asyncio.CancelledError() + with ( + mock.patch.object(resource, "retrieve", return_value=make_file("uploaded")), + mock.patch.object(resource, "_sleep", side_effect=cancellation), + ): + with pytest.raises(asyncio.CancelledError) as caught: + await resource.wait_for_processing(FILE_ID) + assert caught.value is cancellation