From 08bd2c6f113d59a6b20840f358243bb8063378cc Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Fri, 10 Apr 2026 16:14:05 -0400 Subject: [PATCH] feat(openai): trace responses api tool spans Record child tool spans for Responses API tool calls so function and web search activity appear separately from the parent LLM span. This keeps tool metadata, inputs, outputs, and errors visible for both standard and streaming responses. Add VCR-backed coverage for function-call and web-search tool spans to lock in the new tracing behavior. --- ...ai_responses_function_call_tool_spans.yaml | 121 ++++++++ ...penai_responses_web_search_tool_spans.yaml | 128 ++++++++ ...esponses_web_search_tool_spans_stream.yaml | 287 ++++++++++++++++++ .../integrations/openai/test_openai.py | 143 +++++++++ .../braintrust/integrations/openai/tracing.py | 145 ++++++++- 5 files changed, 822 insertions(+), 2 deletions(-) create mode 100644 py/src/braintrust/integrations/openai/cassettes/test_openai_responses_function_call_tool_spans.yaml create mode 100644 py/src/braintrust/integrations/openai/cassettes/test_openai_responses_web_search_tool_spans.yaml create mode 100644 py/src/braintrust/integrations/openai/cassettes/test_openai_responses_web_search_tool_spans_stream.yaml diff --git a/py/src/braintrust/integrations/openai/cassettes/test_openai_responses_function_call_tool_spans.yaml b/py/src/braintrust/integrations/openai/cassettes/test_openai_responses_function_call_tool_spans.yaml new file mode 100644 index 00000000..d92dd00e --- /dev/null +++ b/py/src/braintrust/integrations/openai/cassettes/test_openai_responses_function_call_tool_spans.yaml @@ -0,0 +1,121 @@ +interactions: +- request: + body: '{"input":"Use the get_weather tool with location Paris. Do not answer directly.","model":"gpt-4.1-mini","tool_choice":{"type":"function","name":"get_weather"},"tools":[{"type":"function","name":"get_weather","description":"Get + the weather for a location.","parameters":{"type":"object","properties":{"location":{"type":"string"}},"required":["location"]}}]}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '357' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.24.0 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 2.24.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.13.3 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: "{\n \"id\": \"resp_07d7ba35639d95650069d956477504819ea8e4311bf5635340\",\n + \ \"object\": \"response\",\n \"created_at\": 1775851079,\n \"status\": + \"completed\",\n \"background\": false,\n \"billing\": {\n \"payer\": + \"developer\"\n },\n \"completed_at\": 1775851083,\n \"error\": null,\n + \ \"frequency_penalty\": 0.0,\n \"incomplete_details\": null,\n \"instructions\": + null,\n \"max_output_tokens\": null,\n \"max_tool_calls\": null,\n \"model\": + \"gpt-4.1-mini-2025-04-14\",\n \"output\": [\n {\n \"id\": \"fc_07d7ba35639d95650069d9564b12b0819eb1622a71fa84ac45\",\n + \ \"type\": \"function_call\",\n \"status\": \"completed\",\n \"arguments\": + \"{\\\"location\\\":\\\"Paris\\\"}\",\n \"call_id\": \"call_o1UO7CPVJ9fm0E4rThpUGTR2\",\n + \ \"name\": \"get_weather\"\n }\n ],\n \"parallel_tool_calls\": true,\n + \ \"presence_penalty\": 0.0,\n \"previous_response_id\": null,\n \"prompt_cache_key\": + null,\n \"prompt_cache_retention\": null,\n \"reasoning\": {\n \"effort\": + null,\n \"summary\": null\n },\n \"safety_identifier\": null,\n \"service_tier\": + \"default\",\n \"store\": true,\n \"temperature\": 1.0,\n \"text\": {\n + \ \"format\": {\n \"type\": \"text\"\n },\n \"verbosity\": \"medium\"\n + \ },\n \"tool_choice\": {\n \"type\": \"function\",\n \"name\": \"get_weather\"\n + \ },\n \"tools\": [\n {\n \"type\": \"function\",\n \"description\": + \"Get the weather for a location.\",\n \"name\": \"get_weather\",\n \"parameters\": + {\n \"type\": \"object\",\n \"properties\": {\n \"location\": + {\n \"type\": \"string\"\n }\n },\n \"required\": + [\n \"location\"\n ],\n \"additionalProperties\": false\n + \ },\n \"strict\": true\n }\n ],\n \"top_logprobs\": 0,\n \"top_p\": + 1.0,\n \"truncation\": \"disabled\",\n \"usage\": {\n \"input_tokens\": + 61,\n \"input_tokens_details\": {\n \"cached_tokens\": 0\n },\n + \ \"output_tokens\": 6,\n \"output_tokens_details\": {\n \"reasoning_tokens\": + 0\n },\n \"total_tokens\": 67\n },\n \"user\": null,\n \"metadata\": + {}\n}" + headers: + CF-RAY: + - 9ea452da883cfcd9-YYZ + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Fri, 10 Apr 2026 19:58:03 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + content-length: + - '1920' + openai-organization: + - braintrust-data + openai-processing-ms: + - '3795' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + set-cookie: + - __cf_bm=EEjYY7pHwntarnkOUh91A_nMX5jMeaVg99fZJ_xXL5A-1775851078.805831-1.0.1.1-.eWi3f4FG_j3WBmIKK7KUh9TGMopRUJE1hw2fPrD3UfdzDzYgc7xTlxXSGb3UYbkX2fW9gM922WxwtNiDh2XMUgHolvbZibH89tSEZ9opfOXBuJugA4quS0Iyr9INDwG; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Fri, 10 Apr 2026 + 20:28:03 GMT + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999722' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_f5759931ad07431fb68e8ffe60b71ab5 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/openai/cassettes/test_openai_responses_web_search_tool_spans.yaml b/py/src/braintrust/integrations/openai/cassettes/test_openai_responses_web_search_tool_spans.yaml new file mode 100644 index 00000000..7aea24b2 --- /dev/null +++ b/py/src/braintrust/integrations/openai/cassettes/test_openai_responses_web_search_tool_spans.yaml @@ -0,0 +1,128 @@ +interactions: +- request: + body: '{"input":"Search the web for the current weather in Paris and answer in + one sentence.","model":"gpt-4.1-mini","tool_choice":{"type":"web_search_preview"},"tools":[{"type":"web_search_preview","search_context_size":"low"}]}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '222' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.24.0 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 2.24.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.13.3 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: "{\n \"id\": \"resp_02ba15fc2fd264b20069d9564b711c819182af700a09fe0c1f\",\n + \ \"object\": \"response\",\n \"created_at\": 1775851083,\n \"status\": + \"completed\",\n \"background\": false,\n \"billing\": {\n \"payer\": + \"developer\"\n },\n \"completed_at\": 1775851085,\n \"error\": null,\n + \ \"frequency_penalty\": 0.0,\n \"incomplete_details\": null,\n \"instructions\": + null,\n \"max_output_tokens\": null,\n \"max_tool_calls\": null,\n \"model\": + \"gpt-4.1-mini-2025-04-14\",\n \"output\": [\n {\n \"id\": \"ws_02ba15fc2fd264b20069d9564b8aa88191b0fce6f6e4a6d0b6\",\n + \ \"type\": \"web_search_call\",\n \"status\": \"completed\",\n \"action\": + {\n \"type\": \"search\",\n \"queries\": [\n \"Search + the web for the current weather in Paris and answer in one sentence.\"\n ],\n + \ \"query\": \"Search the web for the current weather in Paris and answer + in one sentence.\"\n }\n },\n {\n \"id\": \"msg_02ba15fc2fd264b20069d9564c85b881919add08d779b2c1a1\",\n + \ \"type\": \"message\",\n \"status\": \"completed\",\n \"content\": + [\n {\n \"type\": \"output_text\",\n \"annotations\": + [],\n \"logprobs\": [],\n \"text\": \"As of 9:58\\u202fPM + on April 10, 2026, in Paris, France, the weather is mostly cloudy with a temperature + of 56\\u00b0F (13\\u00b0C).\\n\\n## Weather for Paris, Paris, France:\\nCurrent + Conditions: Mostly cloudy, 56\\u00b0F (13\\u00b0C)\\n\\nDaily Forecast:\\n* + Friday, April 10: Low: 49\\u00b0F (10\\u00b0C), High: 61\\u00b0F (16\\u00b0C), + Description: Cooler with times of clouds and sun\\n* Saturday, April 11: Low: + 43\\u00b0F (6\\u00b0C), High: 61\\u00b0F (16\\u00b0C), Description: Mostly + cloudy and cooler; occasional afternoon rain and drizzle\\n* Sunday, April + 12: Low: 45\\u00b0F (7\\u00b0C), High: 59\\u00b0F (15\\u00b0C), Description: + Mostly cloudy\\n* Monday, April 13: Low: 44\\u00b0F (6\\u00b0C), High: 60\\u00b0F + (15\\u00b0C), Description: A passing shower in the morning; otherwise, cloudy\\n* + Tuesday, April 14: Low: 47\\u00b0F (8\\u00b0C), High: 64\\u00b0F (18\\u00b0C), + Description: Sunny to partly cloudy\\n* Wednesday, April 15: Low: 50\\u00b0F + (10\\u00b0C), High: 67\\u00b0F (19\\u00b0C), Description: Thickening clouds\\n* + Thursday, April 16: Low: 47\\u00b0F (8\\u00b0C), High: 68\\u00b0F (20\\u00b0C), + Description: Nice with intervals of clouds and sunshine\\n \"\n }\n + \ ],\n \"role\": \"assistant\"\n }\n ],\n \"parallel_tool_calls\": + true,\n \"presence_penalty\": 0.0,\n \"previous_response_id\": null,\n \"prompt_cache_key\": + null,\n \"prompt_cache_retention\": null,\n \"reasoning\": {\n \"effort\": + null,\n \"summary\": null\n },\n \"safety_identifier\": null,\n \"service_tier\": + \"auto\",\n \"store\": true,\n \"temperature\": 1.0,\n \"text\": {\n \"format\": + {\n \"type\": \"text\"\n },\n \"verbosity\": \"medium\"\n },\n + \ \"tool_choice\": {\n \"type\": \"web_search_preview\"\n },\n \"tools\": + [\n {\n \"type\": \"web_search_preview\",\n \"search_context_size\": + \"low\",\n \"user_location\": {\n \"type\": \"approximate\",\n + \ \"city\": null,\n \"country\": \"US\",\n \"region\": + null,\n \"timezone\": null\n }\n }\n ],\n \"top_logprobs\": + 0,\n \"top_p\": 1.0,\n \"truncation\": \"disabled\",\n \"usage\": {\n \"input_tokens\": + 323,\n \"input_tokens_details\": {\n \"cached_tokens\": 0\n },\n + \ \"output_tokens\": 303,\n \"output_tokens_details\": {\n \"reasoning_tokens\": + 0\n },\n \"total_tokens\": 626\n },\n \"user\": null,\n \"metadata\": + {}\n}" + headers: + CF-RAY: + - 9ea452f74992ac6c-YYZ + Connection: + - keep-alive + Content-Type: + - application/json + Date: + - Fri, 10 Apr 2026 19:58:05 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + content-length: + - '3357' + openai-organization: + - braintrust-data + openai-processing-ms: + - '1870' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + set-cookie: + - __cf_bm=74yoaPxjLsemgt3Gm2pbBidFEZrSluN1vCX7nOSZM4E-1775851083.4086235-1.0.1.1-5hp2jlDwjwyIsWI_AqKiWhK0a47Xwpx4l6KxT2tkPUDtBzOODfHxSoHdNw6FbtRw2knfc1s8QlkMK7r03gOBIb5.o0f_oP83TNk8gUXTjO1WvAQTKvzBAFgADElEFi1v; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Fri, 10 Apr 2026 + 20:28:05 GMT + x-request-id: + - req_0c2c210a4f5e4e8a9406206d1f4e7796 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/openai/cassettes/test_openai_responses_web_search_tool_spans_stream.yaml b/py/src/braintrust/integrations/openai/cassettes/test_openai_responses_web_search_tool_spans_stream.yaml new file mode 100644 index 00000000..aadfc0bc --- /dev/null +++ b/py/src/braintrust/integrations/openai/cassettes/test_openai_responses_web_search_tool_spans_stream.yaml @@ -0,0 +1,287 @@ +interactions: +- request: + body: '{"input":"Search the web for the latest weather in Paris and answer briefly.","model":"gpt-4.1-mini","stream":true,"tool_choice":{"type":"web_search_preview"},"tools":[{"type":"web_search_preview","search_context_size":"low"}]}' + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '227' + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 2.24.0 + X-Stainless-Arch: + - arm64 + X-Stainless-Async: + - 'false' + X-Stainless-Lang: + - python + X-Stainless-OS: + - MacOS + X-Stainless-Package-Version: + - 2.24.0 + X-Stainless-Runtime: + - CPython + X-Stainless-Runtime-Version: + - 3.13.3 + x-stainless-read-timeout: + - '600' + x-stainless-retry-count: + - '0' + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_03f5d042dac8207a0069d9564d796081a2921aedf0fc2b4d3c\",\"object\":\"response\",\"created_at\":1775851085,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-4.1-mini-2025-04-14\",\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":null,\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":true,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":{\"type\":\"web_search_preview\"},\"tools\":[{\"type\":\"web_search_preview\",\"search_context_size\":\"low\",\"user_location\":{\"type\":\"approximate\",\"city\":null,\"country\":\"US\",\"region\":null,\"timezone\":null}}],\"top_logprobs\":0,\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":0}\n\nevent: + response.in_progress\ndata: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_03f5d042dac8207a0069d9564d796081a2921aedf0fc2b4d3c\",\"object\":\"response\",\"created_at\":1775851085,\"status\":\"in_progress\",\"background\":false,\"completed_at\":null,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-4.1-mini-2025-04-14\",\"output\":[],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":null,\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":true,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":{\"type\":\"web_search_preview\"},\"tools\":[{\"type\":\"web_search_preview\",\"search_context_size\":\"low\",\"user_location\":{\"type\":\"approximate\",\"city\":null,\"country\":\"US\",\"region\":null,\"timezone\":null}}],\"top_logprobs\":0,\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":null,\"user\":null,\"metadata\":{}},\"sequence_number\":1}\n\nevent: + response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"ws_03f5d042dac8207a0069d9564d8b9081a293302d2527ab9b4c\",\"type\":\"web_search_call\",\"status\":\"in_progress\",\"action\":{\"type\":\"search\"}},\"output_index\":0,\"sequence_number\":2}\n\nevent: + response.web_search_call.in_progress\ndata: {\"type\":\"response.web_search_call.in_progress\",\"item_id\":\"ws_03f5d042dac8207a0069d9564d8b9081a293302d2527ab9b4c\",\"output_index\":0,\"sequence_number\":3}\n\nevent: + response.web_search_call.searching\ndata: {\"type\":\"response.web_search_call.searching\",\"item_id\":\"ws_03f5d042dac8207a0069d9564d8b9081a293302d2527ab9b4c\",\"output_index\":0,\"sequence_number\":4}\n\nevent: + response.web_search_call.completed\ndata: {\"type\":\"response.web_search_call.completed\",\"item_id\":\"ws_03f5d042dac8207a0069d9564d8b9081a293302d2527ab9b4c\",\"output_index\":0,\"sequence_number\":5}\n\nevent: + response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"ws_03f5d042dac8207a0069d9564d8b9081a293302d2527ab9b4c\",\"type\":\"web_search_call\",\"status\":\"completed\",\"action\":{\"type\":\"search\",\"queries\":[\"Search + the web for the latest weather in Paris and answer briefly.\"],\"query\":\"Search + the web for the latest weather in Paris and answer briefly.\"}},\"output_index\":0,\"sequence_number\":6}\n\nevent: + response.output_item.added\ndata: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"type\":\"message\",\"status\":\"in_progress\",\"content\":[],\"role\":\"assistant\"},\"output_index\":1,\"sequence_number\":7}\n\nevent: + response.content_part.added\ndata: {\"type\":\"response.content_part.added\",\"content_index\":0,\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"output_index\":1,\"part\":{\"type\":\"output_text\",\"annotations\":[],\"logprobs\":[],\"text\":\"\"},\"sequence_number\":8}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"As\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"oXtqGgNWuXairA\",\"output_index\":1,\"sequence_number\":9}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + of\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"DezMVL0zQ5JOi\",\"output_index\":1,\"sequence_number\":10}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + \",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"z5BN44pxqz4lVrc\",\"output_index\":1,\"sequence_number\":11}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"9\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"3AcemChdpNnBDOP\",\"output_index\":1,\"sequence_number\":12}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\":\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"T1Pmkm2CopLm529\",\"output_index\":1,\"sequence_number\":13}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"58\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"R3Ef5QZ3I9nHdu\",\"output_index\":1,\"sequence_number\":14}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + PM\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"FCc5hVpUqrdpx\",\"output_index\":1,\"sequence_number\":15}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + on\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"GRCMRZtuuFieo\",\"output_index\":1,\"sequence_number\":16}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + April\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"y9dhsZTnR3\",\"output_index\":1,\"sequence_number\":17}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + \",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"rjewnmAHjO23BRh\",\"output_index\":1,\"sequence_number\":18}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"10\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"BAKP1yaKGS10dg\",\"output_index\":1,\"sequence_number\":19}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\",\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"ui8hpp01CaxeXjd\",\"output_index\":1,\"sequence_number\":20}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + \",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"sDCD23uxk1gjZNI\",\"output_index\":1,\"sequence_number\":21}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"202\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"oH17vbjPR3Gxo\",\"output_index\":1,\"sequence_number\":22}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"6\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"A73sByVSLHe47oA\",\"output_index\":1,\"sequence_number\":23}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\",\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"1cBsBVHd7e7GBje\",\"output_index\":1,\"sequence_number\":24}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + in\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"UqcHFyOz8EZ7i\",\"output_index\":1,\"sequence_number\":25}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + Paris\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"dDLGNGDA4l\",\"output_index\":1,\"sequence_number\":26}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\",\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"JvdL46w7xpPO4Qc\",\"output_index\":1,\"sequence_number\":27}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + France\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"6KrfTqB3W\",\"output_index\":1,\"sequence_number\":28}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\",\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"fKxpKd3NRnCDyBC\",\"output_index\":1,\"sequence_number\":29}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + the\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"NkMs5P1XA8m9\",\"output_index\":1,\"sequence_number\":30}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + current\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"pGHCWArz\",\"output_index\":1,\"sequence_number\":31}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + weather\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"G3xua51D\",\"output_index\":1,\"sequence_number\":32}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + is\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"gmrENVyX4D4No\",\"output_index\":1,\"sequence_number\":33}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + mostly\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"8cZ0ybSLf\",\"output_index\":1,\"sequence_number\":34}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + cloudy\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"IrAjjruxU\",\"output_index\":1,\"sequence_number\":35}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + with\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"DhwOkZ1SmIR\",\"output_index\":1,\"sequence_number\":36}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + a\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"t1QP6ovxMhLAWK\",\"output_index\":1,\"sequence_number\":37}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + temperature\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"m3t6\",\"output_index\":1,\"sequence_number\":38}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + of\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"XiEYrkOBeOlwv\",\"output_index\":1,\"sequence_number\":39}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + \",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"t1ylYcHvXAklTx6\",\"output_index\":1,\"sequence_number\":40}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"56\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"tcQwOdZ3wrPt13\",\"output_index\":1,\"sequence_number\":41}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"\xB0F\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"hlQtWozKwXNuDn\",\"output_index\":1,\"sequence_number\":42}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + (\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"wp3FmivMogtTRw\",\"output_index\":1,\"sequence_number\":43}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"13\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"gXnOQUSB70JL6Y\",\"output_index\":1,\"sequence_number\":44}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"\xB0C\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"PwLvLnbHzYhYRR\",\"output_index\":1,\"sequence_number\":45}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\").\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"W8GnwmHwKZn2xj\",\"output_index\":1,\"sequence_number\":46}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"\\n\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"OrHTOvSki9XQKtf\",\"output_index\":1,\"sequence_number\":47}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"\\n\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"9AtfUAEHcBpBIRk\",\"output_index\":1,\"sequence_number\":48}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"## + Weather for Paris, Paris, France:\\nCurrent Conditions: Mostly cloudy, 56\xB0F + (13\xB0C)\\n\\nDaily Forecast:\\n* Friday, April 10: Low: 49\xB0F (10\xB0C), + High: 61\xB0F (16\xB0C), Description: Cooler with times of clouds and sun\\n* + Saturday, April 11: Low: 43\xB0F (6\xB0C), High: 61\xB0F (16\xB0C), Description: + Mostly cloudy and cooler; occasional afternoon rain and drizzle\\n* Sunday, + April 12: Low: 45\xB0F (7\xB0C), High: 59\xB0F (15\xB0C), Description: Mostly + cloudy\\n* Monday, April 13: Low: 44\xB0F (6\xB0C), High: 60\xB0F (15\xB0C), + Description: A passing shower in the morning; otherwise, cloudy\\n* Tuesday, + April 14: Low: 47\xB0F (8\xB0C), High: 64\xB0F (18\xB0C), Description: Sunny + to partly cloudy\\n* Wednesday, April 15: Low: 50\xB0F (10\xB0C), High: 67\xB0F + (19\xB0C), Description: Thickening clouds\\n* Thursday, April 16: Low: 47\xB0F + (8\xB0C), High: 68\xB0F (20\xB0C), Description: Nice with intervals of clouds + and sunshine\\n\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"Xklpf\",\"output_index\":1,\"sequence_number\":49}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"\\n\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"BbgnQqGbVpq3FHg\",\"output_index\":1,\"sequence_number\":50}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"\\n\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"LS4xlFWfprzxJM0\",\"output_index\":1,\"sequence_number\":51}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"Please\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"bK1SMF0pgR\",\"output_index\":1,\"sequence_number\":52}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + note\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"9RaVJi5lM7u\",\"output_index\":1,\"sequence_number\":53}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + that\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"LiNLmufsoD4\",\"output_index\":1,\"sequence_number\":54}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + thunderstorms\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"nA\",\"output_index\":1,\"sequence_number\":55}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + are\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"rAum51jjIlOJ\",\"output_index\":1,\"sequence_number\":56}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + expected\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"SlWo26Y\",\"output_index\":1,\"sequence_number\":57}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + across\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"Fx31Tb9CG\",\"output_index\":1,\"sequence_number\":58}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + France\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"zvk1jJ7tD\",\"output_index\":1,\"sequence_number\":59}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + this\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"qCcLhNj2Qkk\",\"output_index\":1,\"sequence_number\":60}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + weekend\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"sH2GroZA\",\"output_index\":1,\"sequence_number\":61}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\",\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"ho7Visk4FrSuMqT\",\"output_index\":1,\"sequence_number\":62}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + bringing\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"lgqz6r6\",\"output_index\":1,\"sequence_number\":63}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + an\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"jQ99QXg9hVWW5\",\"output_index\":1,\"sequence_number\":64}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + end\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"5cNy3jLoIyAS\",\"output_index\":1,\"sequence_number\":65}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + to\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"RdmWqsuVJ61vo\",\"output_index\":1,\"sequence_number\":66}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + the\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"s6PgPK4M1e8U\",\"output_index\":1,\"sequence_number\":67}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + recent\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"VWqgi3wqG\",\"output_index\":1,\"sequence_number\":68}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + warm\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"yCqeyMyAdl6\",\"output_index\":1,\"sequence_number\":69}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + conditions\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"Pb4oR\",\"output_index\":1,\"sequence_number\":70}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\".\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"foQWXSzdKEyMHCG\",\"output_index\":1,\"sequence_number\":71}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + \",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"yMrCFPACRtHqQAs\",\"output_index\":1,\"sequence_number\":72}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\"([connexionfrance.com](https://www.connexionfrance.com/news/storms-expected-across-france-this-weekend-following-sunny-spell/782631?utm_source=openai))\",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"aCep4nPac\",\"output_index\":1,\"sequence_number\":73}\n\nevent: + response.output_text.annotation.added\ndata: {\"type\":\"response.output_text.annotation.added\",\"annotation\":{\"type\":\"url_citation\",\"end_index\":1239,\"start_index\":1088,\"title\":\"Thunderstorms + to hit France this weekend as temperatures drop\",\"url\":\"https://www.connexionfrance.com/news/storms-expected-across-france-this-weekend-following-sunny-spell/782631?utm_source=openai\"},\"annotation_index\":0,\"content_index\":0,\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"output_index\":1,\"sequence_number\":74}\n\nevent: + response.output_text.delta\ndata: {\"type\":\"response.output_text.delta\",\"content_index\":0,\"delta\":\" + \",\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"obfuscation\":\"0jd9jzGYIoe70ES\",\"output_index\":1,\"sequence_number\":75}\n\nevent: + response.output_text.done\ndata: {\"type\":\"response.output_text.done\",\"content_index\":0,\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"logprobs\":[],\"output_index\":1,\"sequence_number\":76,\"text\":\"As + of 9:58 PM on April 10, 2026, in Paris, France, the current weather is mostly + cloudy with a temperature of 56\xB0F (13\xB0C).\\n\\n## Weather for Paris, + Paris, France:\\nCurrent Conditions: Mostly cloudy, 56\xB0F (13\xB0C)\\n\\nDaily + Forecast:\\n* Friday, April 10: Low: 49\xB0F (10\xB0C), High: 61\xB0F (16\xB0C), + Description: Cooler with times of clouds and sun\\n* Saturday, April 11: Low: + 43\xB0F (6\xB0C), High: 61\xB0F (16\xB0C), Description: Mostly cloudy and + cooler; occasional afternoon rain and drizzle\\n* Sunday, April 12: Low: 45\xB0F + (7\xB0C), High: 59\xB0F (15\xB0C), Description: Mostly cloudy\\n* Monday, + April 13: Low: 44\xB0F (6\xB0C), High: 60\xB0F (15\xB0C), Description: A passing + shower in the morning; otherwise, cloudy\\n* Tuesday, April 14: Low: 47\xB0F + (8\xB0C), High: 64\xB0F (18\xB0C), Description: Sunny to partly cloudy\\n* + Wednesday, April 15: Low: 50\xB0F (10\xB0C), High: 67\xB0F (19\xB0C), Description: + Thickening clouds\\n* Thursday, April 16: Low: 47\xB0F (8\xB0C), High: 68\xB0F + (20\xB0C), Description: Nice with intervals of clouds and sunshine\\n\\n\\nPlease + note that thunderstorms are expected across France this weekend, bringing + an end to the recent warm conditions. ([connexionfrance.com](https://www.connexionfrance.com/news/storms-expected-across-france-this-weekend-following-sunny-spell/782631?utm_source=openai)) + \"}\n\nevent: response.content_part.done\ndata: {\"type\":\"response.content_part.done\",\"content_index\":0,\"item_id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"output_index\":1,\"part\":{\"type\":\"output_text\",\"annotations\":[{\"type\":\"url_citation\",\"end_index\":1239,\"start_index\":1088,\"title\":\"Thunderstorms + to hit France this weekend as temperatures drop\",\"url\":\"https://www.connexionfrance.com/news/storms-expected-across-france-this-weekend-following-sunny-spell/782631?utm_source=openai\"}],\"logprobs\":[],\"text\":\"As + of 9:58 PM on April 10, 2026, in Paris, France, the current weather is mostly + cloudy with a temperature of 56\xB0F (13\xB0C).\\n\\n## Weather for Paris, + Paris, France:\\nCurrent Conditions: Mostly cloudy, 56\xB0F (13\xB0C)\\n\\nDaily + Forecast:\\n* Friday, April 10: Low: 49\xB0F (10\xB0C), High: 61\xB0F (16\xB0C), + Description: Cooler with times of clouds and sun\\n* Saturday, April 11: Low: + 43\xB0F (6\xB0C), High: 61\xB0F (16\xB0C), Description: Mostly cloudy and + cooler; occasional afternoon rain and drizzle\\n* Sunday, April 12: Low: 45\xB0F + (7\xB0C), High: 59\xB0F (15\xB0C), Description: Mostly cloudy\\n* Monday, + April 13: Low: 44\xB0F (6\xB0C), High: 60\xB0F (15\xB0C), Description: A passing + shower in the morning; otherwise, cloudy\\n* Tuesday, April 14: Low: 47\xB0F + (8\xB0C), High: 64\xB0F (18\xB0C), Description: Sunny to partly cloudy\\n* + Wednesday, April 15: Low: 50\xB0F (10\xB0C), High: 67\xB0F (19\xB0C), Description: + Thickening clouds\\n* Thursday, April 16: Low: 47\xB0F (8\xB0C), High: 68\xB0F + (20\xB0C), Description: Nice with intervals of clouds and sunshine\\n\\n\\nPlease + note that thunderstorms are expected across France this weekend, bringing + an end to the recent warm conditions. ([connexionfrance.com](https://www.connexionfrance.com/news/storms-expected-across-france-this-weekend-following-sunny-spell/782631?utm_source=openai)) + \"},\"sequence_number\":77}\n\nevent: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[{\"type\":\"url_citation\",\"end_index\":1239,\"start_index\":1088,\"title\":\"Thunderstorms + to hit France this weekend as temperatures drop\",\"url\":\"https://www.connexionfrance.com/news/storms-expected-across-france-this-weekend-following-sunny-spell/782631?utm_source=openai\"}],\"logprobs\":[],\"text\":\"As + of 9:58 PM on April 10, 2026, in Paris, France, the current weather is mostly + cloudy with a temperature of 56\xB0F (13\xB0C).\\n\\n## Weather for Paris, + Paris, France:\\nCurrent Conditions: Mostly cloudy, 56\xB0F (13\xB0C)\\n\\nDaily + Forecast:\\n* Friday, April 10: Low: 49\xB0F (10\xB0C), High: 61\xB0F (16\xB0C), + Description: Cooler with times of clouds and sun\\n* Saturday, April 11: Low: + 43\xB0F (6\xB0C), High: 61\xB0F (16\xB0C), Description: Mostly cloudy and + cooler; occasional afternoon rain and drizzle\\n* Sunday, April 12: Low: 45\xB0F + (7\xB0C), High: 59\xB0F (15\xB0C), Description: Mostly cloudy\\n* Monday, + April 13: Low: 44\xB0F (6\xB0C), High: 60\xB0F (15\xB0C), Description: A passing + shower in the morning; otherwise, cloudy\\n* Tuesday, April 14: Low: 47\xB0F + (8\xB0C), High: 64\xB0F (18\xB0C), Description: Sunny to partly cloudy\\n* + Wednesday, April 15: Low: 50\xB0F (10\xB0C), High: 67\xB0F (19\xB0C), Description: + Thickening clouds\\n* Thursday, April 16: Low: 47\xB0F (8\xB0C), High: 68\xB0F + (20\xB0C), Description: Nice with intervals of clouds and sunshine\\n\\n\\nPlease + note that thunderstorms are expected across France this weekend, bringing + an end to the recent warm conditions. ([connexionfrance.com](https://www.connexionfrance.com/news/storms-expected-across-france-this-weekend-following-sunny-spell/782631?utm_source=openai)) + \"}],\"role\":\"assistant\"},\"output_index\":1,\"sequence_number\":78}\n\nevent: + response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_03f5d042dac8207a0069d9564d796081a2921aedf0fc2b4d3c\",\"object\":\"response\",\"created_at\":1775851085,\"status\":\"completed\",\"background\":false,\"completed_at\":1775851086,\"error\":null,\"frequency_penalty\":0.0,\"incomplete_details\":null,\"instructions\":null,\"max_output_tokens\":null,\"max_tool_calls\":null,\"model\":\"gpt-4.1-mini-2025-04-14\",\"output\":[{\"id\":\"ws_03f5d042dac8207a0069d9564d8b9081a293302d2527ab9b4c\",\"type\":\"web_search_call\",\"status\":\"completed\",\"action\":{\"type\":\"search\",\"queries\":[\"Search + the web for the latest weather in Paris and answer briefly.\"],\"query\":\"Search + the web for the latest weather in Paris and answer briefly.\"}},{\"id\":\"msg_03f5d042dac8207a0069d9564e380881a29cc70ad116ba2de3\",\"type\":\"message\",\"status\":\"completed\",\"content\":[{\"type\":\"output_text\",\"annotations\":[{\"type\":\"url_citation\",\"end_index\":1239,\"start_index\":1088,\"title\":\"Thunderstorms + to hit France this weekend as temperatures drop\",\"url\":\"https://www.connexionfrance.com/news/storms-expected-across-france-this-weekend-following-sunny-spell/782631?utm_source=openai\"}],\"logprobs\":[],\"text\":\"As + of 9:58 PM on April 10, 2026, in Paris, France, the current weather is mostly + cloudy with a temperature of 56\xB0F (13\xB0C).\\n\\n## Weather for Paris, + Paris, France:\\nCurrent Conditions: Mostly cloudy, 56\xB0F (13\xB0C)\\n\\nDaily + Forecast:\\n* Friday, April 10: Low: 49\xB0F (10\xB0C), High: 61\xB0F (16\xB0C), + Description: Cooler with times of clouds and sun\\n* Saturday, April 11: Low: + 43\xB0F (6\xB0C), High: 61\xB0F (16\xB0C), Description: Mostly cloudy and + cooler; occasional afternoon rain and drizzle\\n* Sunday, April 12: Low: 45\xB0F + (7\xB0C), High: 59\xB0F (15\xB0C), Description: Mostly cloudy\\n* Monday, + April 13: Low: 44\xB0F (6\xB0C), High: 60\xB0F (15\xB0C), Description: A passing + shower in the morning; otherwise, cloudy\\n* Tuesday, April 14: Low: 47\xB0F + (8\xB0C), High: 64\xB0F (18\xB0C), Description: Sunny to partly cloudy\\n* + Wednesday, April 15: Low: 50\xB0F (10\xB0C), High: 67\xB0F (19\xB0C), Description: + Thickening clouds\\n* Thursday, April 16: Low: 47\xB0F (8\xB0C), High: 68\xB0F + (20\xB0C), Description: Nice with intervals of clouds and sunshine\\n\\n\\nPlease + note that thunderstorms are expected across France this weekend, bringing + an end to the recent warm conditions. ([connexionfrance.com](https://www.connexionfrance.com/news/storms-expected-across-france-this-weekend-following-sunny-spell/782631?utm_source=openai)) + \"}],\"role\":\"assistant\"}],\"parallel_tool_calls\":true,\"presence_penalty\":0.0,\"previous_response_id\":null,\"prompt_cache_key\":null,\"prompt_cache_retention\":null,\"reasoning\":{\"effort\":null,\"summary\":null},\"safety_identifier\":null,\"service_tier\":\"auto\",\"store\":true,\"temperature\":1.0,\"text\":{\"format\":{\"type\":\"text\"},\"verbosity\":\"medium\"},\"tool_choice\":{\"type\":\"web_search_preview\"},\"tools\":[{\"type\":\"web_search_preview\",\"search_context_size\":\"low\",\"user_location\":{\"type\":\"approximate\",\"city\":null,\"country\":\"US\",\"region\":null,\"timezone\":null}}],\"top_logprobs\":0,\"top_p\":1.0,\"truncation\":\"disabled\",\"usage\":{\"input_tokens\":319,\"input_tokens_details\":{\"cached_tokens\":0},\"output_tokens\":365,\"output_tokens_details\":{\"reasoning_tokens\":0},\"total_tokens\":684},\"user\":null,\"metadata\":{}},\"sequence_number\":79}\n\n" + headers: + CF-RAY: + - 9ea45303da40aafe-YYZ + Connection: + - keep-alive + Content-Type: + - text/event-stream; charset=utf-8 + Date: + - Fri, 10 Apr 2026 19:58:05 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + openai-organization: + - braintrust-data + openai-processing-ms: + - '64' + openai-project: + - proj_vsCSXafhhByzWOThMrJcZiw9 + openai-version: + - '2020-10-01' + set-cookie: + - __cf_bm=cWCcKWE0Gw_p43_GGOzMDB.0IBkpbqd8s7VD52iS1PI-1775851085.4208076-1.0.1.1-OSuK3eiEFcSNlp.j7BXM8DHvjA_QdDSMaaoV8nftCjl8EIxE2yYZJiLTO0_OLTiZiv5yCAZvHO1GQDDXbA6sXOIu.N.GDGvYrxGgYB8_YvAnqW0h_WDFpeR7t1yAaIDg; + HttpOnly; Secure; Path=/; Domain=api.openai.com; Expires=Fri, 10 Apr 2026 + 20:28:05 GMT + x-request-id: + - req_a3a835d54a279a3cb374e32d69a00821 + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/openai/test_openai.py b/py/src/braintrust/integrations/openai/test_openai.py index 516ab777..d80b1c8a 100644 --- a/py/src/braintrust/integrations/openai/test_openai.py +++ b/py/src/braintrust/integrations/openai/test_openai.py @@ -15,6 +15,7 @@ ChatCompletionWrapper, _materialize_logged_file_input, ) +from braintrust.span_types import SpanTypeAttribute from braintrust.test_helpers import assert_dict_matches, init_test_logger from braintrust.wrappers.test_utils import assert_metrics_are_valid, verify_autoinstrument_script from openai import AsyncOpenAI @@ -25,6 +26,7 @@ TEST_ORG_ID = "test-org-openai-py-tracing" PROJECT_NAME = "test-project-openai-py-tracing" TEST_MODEL = "gpt-4o-mini" # cheapest model for tests +RESPONSES_TOOL_MODEL = "gpt-4.1-mini" TEST_PROMPT = "What's 12 + 12?" TEST_SYSTEM_PROMPT = "You are a helpful assistant that only responds with numbers." @@ -36,6 +38,34 @@ def memory_logger(): yield bgl +def _find_spans_by_type(spans, span_type): + return [span for span in spans if span["span_attributes"]["type"] == span_type] + + +def _find_span_by_name(spans, name): + return next(span for span in spans if span["span_attributes"]["name"] == name) + + +def _supports_response_function_tools() -> bool: + try: + from openai.types.responses import ResponseFunctionToolCall + + del ResponseFunctionToolCall + except ImportError: + return False + return True + + +def _supports_response_web_search_tools() -> bool: + try: + from openai.types.responses import ResponseFunctionWebSearch, WebSearchPreviewToolParam + + del ResponseFunctionWebSearch, WebSearchPreviewToolParam + except ImportError: + return False + return True + + @pytest.mark.vcr def test_openai_chat_metrics(memory_logger): assert not memory_logger.pop() @@ -259,6 +289,119 @@ class SimpleAnswer(BaseModel): assert_metrics_are_valid(metrics, start, end) +@pytest.mark.vcr +def test_openai_responses_function_call_tool_spans(memory_logger): + if not _supports_response_function_tools(): + pytest.skip("Responses function tool calls are not available in this SDK version") + + assert not memory_logger.pop() + + client = wrap_openai(openai.OpenAI()) + response = client.responses.create( + model=RESPONSES_TOOL_MODEL, + input="Use the get_weather tool with location Paris. Do not answer directly.", + tools=[ + { + "type": "function", + "name": "get_weather", + "description": "Get the weather for a location.", + "parameters": { + "type": "object", + "properties": {"location": {"type": "string"}}, + "required": ["location"], + }, + } + ], + tool_choice={"type": "function", "name": "get_weather"}, + ) + + function_call = next(output for output in response.output if getattr(output, "type", None) == "function_call") + assert function_call.name == "get_weather" + assert "Paris" in function_call.arguments + + spans = memory_logger.pop() + llm_spans = _find_spans_by_type(spans, SpanTypeAttribute.LLM) + tool_spans = _find_spans_by_type(spans, SpanTypeAttribute.TOOL) + + assert len(llm_spans) == 1 + tool_span = _find_span_by_name(tool_spans, "get_weather") + assert tool_span["span_parents"] == [llm_spans[0]["span_id"]] + assert tool_span["metadata"]["tool_type"] == "function_call" + assert tool_span["metadata"]["call_id"] == function_call.call_id + assert "Paris" in str(tool_span["input"]) + + +@pytest.mark.vcr +def test_openai_responses_web_search_tool_spans(memory_logger): + if not _supports_response_web_search_tools(): + pytest.skip("Responses web search tools are not available in this SDK version") + + assert not memory_logger.pop() + + client = wrap_openai(openai.OpenAI()) + response = client.responses.create( + model=RESPONSES_TOOL_MODEL, + input="Search the web for the current weather in Paris and answer in one sentence.", + tools=[{"type": "web_search_preview", "search_context_size": "low"}], + tool_choice={"type": "web_search_preview"}, + ) + + web_search_call = next(output for output in response.output if getattr(output, "type", None) == "web_search_call") + assert getattr(web_search_call, "status", None) + assert response.output_text + + spans = memory_logger.pop() + llm_spans = _find_spans_by_type(spans, SpanTypeAttribute.LLM) + tool_spans = _find_spans_by_type(spans, SpanTypeAttribute.TOOL) + + assert len(llm_spans) == 1 + tool_span = _find_span_by_name(tool_spans, "web_search_call") + assert tool_span["span_parents"] == [llm_spans[0]["span_id"]] + assert tool_span["metadata"]["tool_type"] == "web_search_call" + assert tool_span["metadata"]["status"] == web_search_call.status + + +@pytest.mark.vcr +def test_openai_responses_web_search_tool_spans_stream(memory_logger): + if not _supports_response_web_search_tools(): + pytest.skip("Responses web search tools are not available in this SDK version") + + client = openai.OpenAI() + if not hasattr(client.responses, "stream"): + pytest.skip("openai.responses.stream is not available in this SDK version") + + assert not memory_logger.pop() + + wrapped_client = wrap_openai(openai.OpenAI()) + with wrapped_client.responses.stream( + model=RESPONSES_TOOL_MODEL, + input="Search the web for the latest weather in Paris and answer briefly.", + tools=[{"type": "web_search_preview", "search_context_size": "low"}], + tool_choice={"type": "web_search_preview"}, + ) as stream: + event_types = [] + for event in stream: + event_types.append(event.type) + final_response = stream.get_final_response() + + assert any(event_type.startswith("response.web_search_call.") for event_type in event_types) + web_search_call = next( + output for output in final_response.output if getattr(output, "type", None) == "web_search_call" + ) + assert final_response.output_text + assert getattr(web_search_call, "status", None) + + spans = memory_logger.pop() + llm_spans = _find_spans_by_type(spans, SpanTypeAttribute.LLM) + tool_spans = _find_spans_by_type(spans, SpanTypeAttribute.TOOL) + + assert len(llm_spans) == 1 + tool_span = _find_span_by_name(tool_spans, "web_search_call") + assert tool_span["span_parents"] == [llm_spans[0]["span_id"]] + assert tool_span["metadata"]["tool_type"] == "web_search_call" + assert tool_span["metadata"]["status"] == web_search_call.status + + @pytest.mark.vcr def test_openai_embeddings(memory_logger): assert not memory_logger.pop() diff --git a/py/src/braintrust/integrations/openai/tracing.py b/py/src/braintrust/integrations/openai/tracing.py index 0f0380d0..764854a3 100644 --- a/py/src/braintrust/integrations/openai/tracing.py +++ b/py/src/braintrust/integrations/openai/tracing.py @@ -2,6 +2,7 @@ import abc import inspect +import json import time from collections.abc import Callable from typing import Any @@ -638,6 +639,140 @@ def parse(self, *args: Any, **kwargs: Any) -> Any: return self._traced_stream +_RESPONSE_TOOL_ITEM_INPUT_KEYS = { + "function_call": ("arguments",), + "web_search_call": ("action",), + "file_search_call": ("queries",), + "code_interpreter_call": ("code", "container_id"), + "computer_call": ("action",), + "image_generation_call": (), + "mcp_call": ("arguments",), +} + + +def _maybe_parse_json_string(value: Any) -> Any: + if not isinstance(value, str): + return value + + stripped = value.strip() + if not stripped or stripped[0] not in "[{": + return value + + try: + return json.loads(stripped) + except json.JSONDecodeError: + return value + + +def _serialize_response_output_items(value: Any) -> list[dict[str, Any]]: + serialized = _try_to_dict(value) + if serialized is None: + return [] + + items = serialized if isinstance(serialized, list) else [serialized] + serialized_items = [] + for item in items: + item_dict = _try_to_dict(item) + if isinstance(item_dict, dict): + serialized_items.append(item_dict) + return serialized_items + + +def _response_tool_span_name(item: dict[str, Any]) -> str: + if item.get("server_label") and item.get("name"): + return f"{item['server_label']}.{item['name']}" + if item.get("name"): + return str(item["name"]) + return str(item.get("type") or "response_tool") + + +def _response_tool_span_input(item: dict[str, Any]) -> Any: + input_keys = _RESPONSE_TOOL_ITEM_INPUT_KEYS.get(item.get("type"), ()) + if not input_keys: + return None + + input_data = clean_nones({key: _maybe_parse_json_string(item.get(key)) for key in input_keys}) + if not input_data: + return None + + if input_keys == ("arguments",): + return input_data["arguments"] + return input_data + + +def _response_tool_span_output(item: dict[str, Any]) -> Any: + if item.get("type") == "function_call": + return None + + excluded_keys = { + "id", + "type", + "name", + "call_id", + "server_label", + "error", + *(_RESPONSE_TOOL_ITEM_INPUT_KEYS.get(item.get("type"), ())), + } + output = clean_nones( + { + key: _maybe_parse_json_string(value) if key in {"output", "error"} else value + for key, value in item.items() + if key not in excluded_keys + } + ) + return output or None + + +def _response_tool_span_error(item: dict[str, Any]) -> Any: + error = item.get("error") + if error is None: + return None + parsed_error = _maybe_parse_json_string(error) + if isinstance(parsed_error, (dict, list)): + return parsed_error + return str(parsed_error) + + +def _response_tool_span_metadata(item: dict[str, Any]) -> dict[str, Any] | None: + return ( + clean_nones( + { + "tool_type": item.get("type"), + "tool_id": item.get("id"), + "call_id": item.get("call_id"), + "status": item.get("status"), + "server_label": item.get("server_label"), + } + ) + or None + ) + + +def _log_response_tool_spans(output: Any, *, parent_export: str | None) -> None: + for item in _serialize_response_output_items(output): + if item.get("type") not in _RESPONSE_TOOL_ITEM_INPUT_KEYS: + continue + + span_args = { + "name": _response_tool_span_name(item), + "type": SpanTypeAttribute.TOOL, + "input": _response_tool_span_input(item), + "metadata": _response_tool_span_metadata(item), + } + if parent_export is not None: + span_args["parent"] = parent_export + + with start_span(**span_args) as tool_span: + error = _response_tool_span_error(item) + if error is not None: + tool_span.log(error=error) + continue + + output_data = _response_tool_span_output(item) + if output_data is not None: + tool_span.log(output=output_data) + + class ResponseWrapper: def __init__( self, @@ -684,7 +819,9 @@ def gen(): all_results.append(item) yield item - span.log(**self._postprocess_streaming_results(all_results)) + event_data = self._postprocess_streaming_results(all_results) + span.log(**event_data) + _log_response_tool_spans(event_data.get("output"), parent_export=span.export()) finally: span.end() @@ -699,6 +836,7 @@ def gen(): event_data["metrics"] = {} event_data["metrics"]["time_to_first_token"] = time.time() - start span.log(**event_data) + _log_response_tool_spans(event_data.get("output"), parent_export=span.export()) return create_response if (raw_requested and hasattr(create_response, "parse")) else raw_response finally: if should_end: @@ -737,7 +875,9 @@ async def gen(): all_results.append(item) yield item - span.log(**self._postprocess_streaming_results(all_results)) + event_data = self._postprocess_streaming_results(all_results) + span.log(**event_data) + _log_response_tool_spans(event_data.get("output"), parent_export=span.export()) finally: span.end() @@ -753,6 +893,7 @@ async def gen(): event_data["metrics"] = {} event_data["metrics"]["time_to_first_token"] = time.time() - start span.log(**event_data) + _log_response_tool_spans(event_data.get("output"), parent_export=span.export()) return create_response if (raw_requested and hasattr(create_response, "parse")) else raw_response finally: if should_end: