diff --git a/.speakeasy/gen.lock b/.speakeasy/gen.lock index 9a8990eb..9ea14791 100644 --- a/.speakeasy/gen.lock +++ b/.speakeasy/gen.lock @@ -1,19 +1,19 @@ lockVersion: 2.0.0 id: c48cf606-fb42-4a45-9c23-8f0555307828 management: - docChecksum: a1446b873bd87da38239da96fef30136 + docChecksum: 28a6d2f2807bdc9333f5dc042fb9e003 docVersion: 1.0.0 speakeasyVersion: 1.787.0 generationVersion: 2.914.0 - releaseVersion: 1.1.11 - configChecksum: 9d5c81b4e0fe110544c084f1cf1dc5b1 + releaseVersion: 1.1.12 + configChecksum: cd9558c65fd6616a3b38c9fad48d8aab repoURL: https://github.com/OpenRouterTeam/python-sdk.git installationURL: https://github.com/OpenRouterTeam/python-sdk.git published: true persistentEdits: - generation_id: 89695304-a652-4c57-99fb-18d5cfc8d261 - pristine_commit_hash: 90a6fb17bc6a68cd7604ec4b9273c4149a97540a - pristine_tree_hash: 5efbf62418627973246e3d236e6dc09685a28f8f + generation_id: a35e8ac7-f059-4ec4-a0e3-88f070a12533 + pristine_commit_hash: 2ebbd6665a75e69bb066ded7a2d78d4b14382e35 + pristine_tree_hash: e0c63b3884b40d36059664bb93ad1723e1825016 features: python: acceptHeaders: 3.0.0 @@ -728,6 +728,10 @@ trackedFiles: id: 73f42e4bc945 last_write_checksum: sha1:599705c7ef520285d8da9e5510109846aa7fd332 pristine_git_object: 539a7c32200bef803260226e81f3225e9529688e + docs/components/benchmarktype.mdx: + id: c191e897ed61 + last_write_checksum: sha1:819054a8e4d3096134355828f732c73e90fa2d67 + pristine_git_object: 2a473fdfeeccc38689147c863d1f857560a97ab4 docs/components/billable.mdx: id: b983fa757056 last_write_checksum: sha1:80d142a13e98a8f13da2850753282adf2ff59521 @@ -5102,20 +5106,28 @@ trackedFiles: pristine_git_object: ea52df6e00e46b6b3b21736ac5cf471f2965d1fb docs/components/unifiedbenchmarksmetasource.mdx: id: c6bc112e85a2 - last_write_checksum: sha1:0ab7afdb2f104c771c5850e05e45f3d116da9704 - pristine_git_object: 1bcb8e017d90d592e5838ad6c09c8bf67cf6d8f0 + last_write_checksum: sha1:50a07e9a95420069bd010b5b8d270fda128d256b + pristine_git_object: 5528a6a2cd6612a404941b73b4e68bb341697e99 docs/components/unifiedbenchmarksmetaversion.mdx: id: 994afa2e9d2c last_write_checksum: sha1:d113d8b5aedeecb78fccd3451e58b42428906a0d pristine_git_object: 23ef4c9a45aea5d918e554f7868f772429ef364c + docs/components/unifiedbenchmarksoritem.mdx: + id: 642fc71c5d26 + last_write_checksum: sha1:5c136345f1f69e6f8db7269a31050aac5ada0894 + pristine_git_object: 99228c0beefaf20bb7b0134946a75404c9d1d0b6 + docs/components/unifiedbenchmarksoritemsource.mdx: + id: b89557fa192a + last_write_checksum: sha1:37dbe5e1fc82f2dd6983383ad74f82048c367062 + pristine_git_object: 558fb378f1e1271e12bbe8777f6d05ef30f15f9b docs/components/unifiedbenchmarksresponse.mdx: id: 713d03de2410 last_write_checksum: sha1:b4145e36aca044cbbf6ee55583979669159d6da0 pristine_git_object: ee48e645716b05d6473f5ba0199fb84264a44d12 docs/components/unifiedbenchmarksresponsedata.mdx: id: fd372e1c57e8 - last_write_checksum: sha1:6065c5a9cec604480837b4018fe94ef1cb6d4b99 - pristine_git_object: eb8045de9612900dea847a76e24965029dad03f4 + last_write_checksum: sha1:303a04665c6586ab8cd3662ff5640d16db8e6c53 + pristine_git_object: ff4c06716df56052ed05ba3a5ac7e4c01d5ea27f docs/components/uniqueinsight.mdx: id: 02fd71bcb47a last_write_checksum: sha1:793e90c78ab450e36533710d3fad6f4a75432e86 @@ -6542,8 +6554,8 @@ trackedFiles: pristine_git_object: ed3d7a92aa56dde65beae92df86cf13120828823 docs/operations/source.mdx: id: 53486739ebf2 - last_write_checksum: sha1:01b8fb26c5211183dee2ba17f235531cba037e5c - pristine_git_object: 20b67f6c8f15960455f78cb1fd44c6d8a065ac84 + last_write_checksum: sha1:ed574764fc3d9c85d55705aee27616e3de0c8fcc + pristine_git_object: 85b44c9e7e9fd56b3a5a0bded755877840cc31b2 docs/operations/subcategory.mdx: id: b3291e6c44b9 last_write_checksum: sha1:2594ecd2302b9515e5997f4fbaaebba4a9a0615c @@ -6694,8 +6706,8 @@ trackedFiles: pristine_git_object: c8541c5c189bae01e80db73fd88767391d7dbb9b docs/sdks/benchmarks/README.mdx: id: 5b483a6770ba - last_write_checksum: sha1:580442b871377a1d5191d9d3d1f30b20667115fe - pristine_git_object: 072d78b5b5aedcdc34724557e76cc0fa03994fba + last_write_checksum: sha1:2d935127bdbf18d1237b3e3af23f9145216806b2 + pristine_git_object: da27ed830fab524c5b9bf2c72031749bb37ed365 docs/sdks/betaanalytics/README.mdx: id: 239279ebf01b last_write_checksum: sha1:5a76d28d77f68efe34ebbcb72850e6a87c74d3b1 @@ -6802,8 +6814,8 @@ trackedFiles: pristine_git_object: 3e38f1a929f7d6b1d6de74604aa87e3d8f010544 pyproject.toml: id: 5d07e7d72637 - last_write_checksum: sha1:68887cf7196f5c439b739b25497108f729e92089 - pristine_git_object: 9ad37b87b230c9153a126f25cb27597e10faac3a + last_write_checksum: sha1:f1ea89560be04f6e082044917ded457ff2ae6fcd + pristine_git_object: 0ac27011d8c3f2dee7ef6b842b31e1a7f98c14de scripts/prepare_readme.py: id: e0c5957a6035 last_write_checksum: sha1:77f44b60b98bc126557ec27391f91dfba764bb54 @@ -6830,8 +6842,8 @@ trackedFiles: pristine_git_object: 86713cfea633e09d33b3d4e65281071fe20e6137 src/openrouter/_version.py: id: d8d15ad6c586 - last_write_checksum: sha1:7881d7cb60e4ba8017a87fdd070b6b4ebd5f81e6 - pristine_git_object: 3d3b3319338c4ded93b69642770503f11927bff5 + last_write_checksum: sha1:d06b3323bcd1fa508ab40232dba1df26d15e3cdd + pristine_git_object: c009a9e6319ae40eeb71248d6552da4a8c73cc44 src/openrouter/analytics.py: id: cb406b5aaabb last_write_checksum: sha1:9e709b71dd0611056dc0cec6150b578defe32841 @@ -6846,8 +6858,8 @@ trackedFiles: pristine_git_object: 7a562e21c7e66c9db666ead83748b73adefe4278 src/openrouter/benchmarks.py: id: 178d2ad8d706 - last_write_checksum: sha1:d9ad86c5125ca8e42125fb886e86583fa3ea5b50 - pristine_git_object: 1b503c89b32b0381c93ee627e74428ae37286e33 + last_write_checksum: sha1:c5a8d0fa780efff4f9f81a12e102cb82d825eb57 + pristine_git_object: 5dcb0e0427dd7e820662fb44bed1099420be25dd src/openrouter/beta.py: id: fffdf54fd8f5 last_write_checksum: sha1:4e34fb96ffe38673ca72e0f8576a8eddd4134d4b @@ -6874,8 +6886,8 @@ trackedFiles: pristine_git_object: ad3d247954547814054c01989a2dff3d12b3e4e1 src/openrouter/components/__init__.py: id: 81754e97b3f4 - last_write_checksum: sha1:c926cc7209a972eafc02ee0697c8c4664131a44d - pristine_git_object: 10eb82a8be06db9cf72d7169b2f82317e29f53a6 + last_write_checksum: sha1:cae2fd824842d8e2f392569cbafa46609fb3afb3 + pristine_git_object: 2656d514ecabf8534e0e625b8a4f977cecb46498 src/openrouter/components/aabenchmarkentry.py: id: e2e0f0b48c82 last_write_checksum: sha1:fab4d9a24d2cea937bb749d46c5f83941e99d65c @@ -8974,12 +8986,16 @@ trackedFiles: pristine_git_object: bc4d5b68d8a9d2e5990b8e663857ee9e26a0e806 src/openrouter/components/unifiedbenchmarksmeta.py: id: ca02b44f633a - last_write_checksum: sha1:2c279a103b3634a6776209e96dc5e66494307cb5 - pristine_git_object: 36d695e2906102ac9fe51e24e8bd80c1e57cccc1 + last_write_checksum: sha1:a7ec056c8b9c73c7f244d03b0cda22fee8263a42 + pristine_git_object: 68d76b7b6dbd1817f825e38902556719b179015d + src/openrouter/components/unifiedbenchmarksoritem.py: + id: 8da361fd40c8 + last_write_checksum: sha1:2f375462570405680ccf18a7e265c3e25fc67dd8 + pristine_git_object: c58ba941246a96890970e6eb1b53588600b2addd src/openrouter/components/unifiedbenchmarksresponse.py: id: 4f7bbccaba03 - last_write_checksum: sha1:19f9e62d134ba7b680d1ddd5a889c879feef169f - pristine_git_object: 7145a0840a470a47f17ea8126012ff6c935e11c3 + last_write_checksum: sha1:2eedc8ad728768805d1bc504caa30989c751ce1f + pristine_git_object: 311cb86d0cc7f42bd9f9842e3783517c182ffe4f src/openrouter/components/unprocessableentityresponseerrordata.py: id: e8ca4a51f994 last_write_checksum: sha1:e97c277b49f4bb6bcfad8eef8a46b382730729a0 @@ -9422,8 +9438,8 @@ trackedFiles: pristine_git_object: a8e00f731e950d64ec8ca7ab10f6fca96884c8e7 src/openrouter/operations/getbenchmarks.py: id: 5fb88644491e - last_write_checksum: sha1:dd712e430db5c813a352a711430da516f17381e6 - pristine_git_object: 493d52d35d2e11943a848c1186605425ee2de8b5 + last_write_checksum: sha1:2df9c948624b4126685e5347e5c9442d2a05945d + pristine_git_object: 62573095f6d816050964d68fb35f61b570acd5d5 src/openrouter/operations/getbyokkey.py: id: d141452bd88a last_write_checksum: sha1:d6d391b230d90f51945144e63e9c53772650bd7b @@ -11226,7 +11242,7 @@ examples: speakeasy-default-get-benchmarks: responses: "200": - application/json: {"data": [{"agentic_index": 58.3, "coding_index": 65.8, "display_name": "GPT-4o", "intelligence_index": 71.2, "model_permaslug": "openai/gpt-4o", "pricing": {"completion": "0.00001", "prompt": "0.0000025"}, "source": "artificial-analysis"}], "meta": {"as_of": "2026-06-03T12:00:00Z", "citation": null, "model_count": 1, "source": null, "source_url": null, "task_type": null, "version": "v1"}} + application/json: {"data": [{"agentic_index": 58.3, "coding_index": 65.8, "display_name": "GPT-4o", "intelligence_index": 71.2, "model_permaslug": "openai/gpt-4o", "pricing": {"completion": "0.00001", "prompt": "0.0000025"}, "source": "artificial-analysis"}, {"accuracy": 0.72, "accuracy_stddev": 0.03, "avg_cost_per_task": 0.002, "benchmark_type": "gpqa_diamond", "display_name": "GPT-4o", "last_run_timestamp": "2026-06-03T12:00:00Z", "model_permaslug": "openai/gpt-4o", "source": "openrouter", "total_tasks": 300}], "meta": {"as_of": "2026-06-03T12:00:00Z", "citation": null, "model_count": 1, "source": null, "source_url": null, "task_type": null, "version": "v1"}} "400": application/json: {"error": {"code": 400, "message": "Invalid request parameters"}} "401": @@ -11337,4 +11353,4 @@ examples: "500": application/json: {"error": {"code": 500, "message": "Internal Server Error"}} examplesVersion: 1.0.2 -releaseNotes: "## Python SDK Changes:\n* `open_router.beta.responses.send()`: \n * `request.plugins[]` **Changed**\n* `open_router.chat.send()`: \n * `request.plugins[]` **Changed**\n* `open_router.presets.create_presets_chat_completions()`: \n * `request.plugins[]` **Changed**\n* `open_router.presets.create_presets_messages()`: \n * `request.plugins[]` **Changed**\n* `open_router.presets.create_presets_responses()`: \n * `request.plugins[]` **Changed**\n* `open_router.responses.send()`: \n * `request.plugins[]` **Changed**\n" +releaseNotes: "## Python SDK Changes:\n* `open_router.benchmarks.get_benchmarks()`: \n * `request.source` **Changed**\n * `response` **Changed**\n" diff --git a/.speakeasy/gen.yaml b/.speakeasy/gen.yaml index 1e3cd1b1..12440489 100644 --- a/.speakeasy/gen.yaml +++ b/.speakeasy/gen.yaml @@ -36,7 +36,7 @@ generation: documentation: mintlify preApplyUnionDiscriminators: true python: - version: 1.1.11 + version: 1.1.12 additionalDependencies: dev: {} main: {} diff --git a/.speakeasy/out.openapi.yaml b/.speakeasy/out.openapi.yaml index a8e4c010..4fa7ead7 100644 --- a/.speakeasy/out.openapi.yaml +++ b/.speakeasy/out.openapi.yaml @@ -22996,6 +22996,7 @@ components: enum: - 'artificial-analysis' - 'design-arena' + - 'openrouter' - null example: 'artificial-analysis' type: @@ -23027,6 +23028,77 @@ components: - 'model_count' - 'task_type' type: 'object' + UnifiedBenchmarksORItem: + example: + accuracy: 0.72 + accuracy_stddev: 0.03 + avg_cost_per_task: 0.002 + benchmark_type: 'gpqa_diamond' + display_name: 'GPT-4o' + last_run_timestamp: '2026-06-03T12:00:00Z' + model_permaslug: 'openai/gpt-4o' + source: 'openrouter' + total_tasks: 300 + properties: + accuracy: + description: 'Aggregate accuracy score from 0 to 1. Higher is better.' + example: 0.72 + format: 'double' + type: 'number' + accuracy_stddev: + description: 'Standard deviation of run accuracy, or null for a single run.' + example: 0.03 + format: 'double' + type: + - 'number' + - 'null' + avg_cost_per_task: + description: 'Average cost per task in USD, or null if unavailable.' + example: 0.002 + format: 'double' + type: + - 'number' + - 'null' + benchmark_type: + description: 'OpenRouter benchmark evaluation type.' + enum: + - 'gpqa_diamond' + - 'tau_bench_verified_airline' + example: 'gpqa_diamond' + type: 'string' + x-speakeasy-unknown-values: allow + display_name: + description: 'Human-readable model name.' + example: 'GPT-4o' + type: 'string' + last_run_timestamp: + description: 'Timestamp of the most recent public benchmark run.' + example: '2026-06-03T12:00:00Z' + type: 'string' + model_permaslug: + description: 'Stable OpenRouter model identifier.' + example: 'openai/gpt-4o' + type: 'string' + source: + description: 'Benchmark source discriminator.' + enum: + - 'openrouter' + type: 'string' + total_tasks: + description: 'Total benchmark tasks across runs.' + example: 300 + type: 'integer' + required: + - 'source' + - 'model_permaslug' + - 'display_name' + - 'benchmark_type' + - 'accuracy' + - 'accuracy_stddev' + - 'avg_cost_per_task' + - 'total_tasks' + - 'last_run_timestamp' + type: 'object' UnifiedBenchmarksResponse: example: data: @@ -23039,6 +23111,15 @@ components: completion: '0.00001' prompt: '0.0000025' source: 'artificial-analysis' + - accuracy: 0.72 + accuracy_stddev: 0.03 + avg_cost_per_task: 0.002 + benchmark_type: 'gpqa_diamond' + display_name: 'GPT-4o' + last_run_timestamp: '2026-06-03T12:00:00Z' + model_permaslug: 'openai/gpt-4o' + source: 'openrouter' + total_tasks: 300 meta: as_of: '2026-06-03T12:00:00Z' citation: null @@ -23054,10 +23135,12 @@ components: mapping: artificial-analysis: '#/components/schemas/UnifiedBenchmarksAAItem' design-arena: '#/components/schemas/UnifiedBenchmarksDAItem' + openrouter: '#/components/schemas/UnifiedBenchmarksORItem' propertyName: 'source' oneOf: - $ref: '#/components/schemas/UnifiedBenchmarksAAItem' - $ref: '#/components/schemas/UnifiedBenchmarksDAItem' + - $ref: '#/components/schemas/UnifiedBenchmarksORItem' type: 'array' meta: $ref: '#/components/schemas/UnifiedBenchmarksMeta' @@ -25833,7 +25916,7 @@ paths: - $ref: "#/components/parameters/AppCategories" /benchmarks: get: - description: 'Unified benchmark endpoint that aggregates scores from multiple benchmark sources (Artificial Analysis, Design Arena). Filter by source to reproduce the exact shapes from the legacy per-source endpoints, or use task_type to find models suited for specific workloads. Authenticate with any valid OpenRouter API key. Rate-limited to 30 requests/minute per key and 500 requests/day per account.' + description: 'Unified benchmark endpoint that aggregates scores from multiple benchmark sources (Artificial Analysis, Design Arena, and OpenRouter''s own tau-bench and GPQA evals). Filter by source to reproduce the exact shapes from the legacy per-source endpoints, or use task_type to find models suited for specific workloads. Authenticate with any valid OpenRouter API key. Rate-limited to 30 requests/minute per key and 500 requests/day per account.' operationId: 'getBenchmarks' parameters: - description: 'Benchmark source to query. Determines the shape of the returned items. When omitted, returns results from all sources.' @@ -25845,6 +25928,7 @@ paths: enum: - 'artificial-analysis' - 'design-arena' + - 'openrouter' example: 'artificial-analysis' type: 'string' x-speakeasy-unknown-values: allow @@ -25906,6 +25990,15 @@ paths: completion: '0.00001' prompt: '0.0000025' source: 'artificial-analysis' + - accuracy: 0.72 + accuracy_stddev: 0.03 + avg_cost_per_task: 0.002 + benchmark_type: 'gpqa_diamond' + display_name: 'GPT-4o' + last_run_timestamp: '2026-06-03T12:00:00Z' + model_permaslug: 'openai/gpt-4o' + source: 'openrouter' + total_tasks: 300 meta: as_of: '2026-06-03T12:00:00Z' citation: null diff --git a/.speakeasy/workflow.lock b/.speakeasy/workflow.lock index ed7b43a3..3654318e 100644 --- a/.speakeasy/workflow.lock +++ b/.speakeasy/workflow.lock @@ -2,8 +2,8 @@ speakeasyVersion: 1.787.0 sources: OpenRouter API: sourceNamespace: open-router-chat-completions-api - sourceRevisionDigest: sha256:3463c7642dac4e0be5ab491342c923d203bec99aaf8f09d978a8773f66aa858f - sourceBlobDigest: sha256:bcf0c7b7716d4ac425c204f04dd3b60a1631a628c87100bdf8171db17ef44797 + sourceRevisionDigest: sha256:84a508bd3595c87832f549d5fc3f1ac5fd440f5165c2995ba3093637555e646c + sourceBlobDigest: sha256:7eefcb515776fd515194da57fd4588ddb6d49151afa71f044f3c485c55730d50 tags: - latest - 1.0.0 @@ -11,10 +11,10 @@ targets: open-router: source: OpenRouter API sourceNamespace: open-router-chat-completions-api - sourceRevisionDigest: sha256:3463c7642dac4e0be5ab491342c923d203bec99aaf8f09d978a8773f66aa858f - sourceBlobDigest: sha256:bcf0c7b7716d4ac425c204f04dd3b60a1631a628c87100bdf8171db17ef44797 + sourceRevisionDigest: sha256:84a508bd3595c87832f549d5fc3f1ac5fd440f5165c2995ba3093637555e646c + sourceBlobDigest: sha256:7eefcb515776fd515194da57fd4588ddb6d49151afa71f044f3c485c55730d50 codeSamplesNamespace: open-router-python-code-samples - codeSamplesRevisionDigest: sha256:0406a0924db3ccfe1fae98f1ee09cfe42acca15d228dfa66c818e954b61a79bf + codeSamplesRevisionDigest: sha256:ebd98981d15a77b71da4a71cd315c29260c58c5db75262df616cd11d56e412d9 workflow: workflowVersion: 1.0.0 speakeasyVersion: 1.787.0 diff --git a/RELEASES.md b/RELEASES.md index ad53c69d..e7525e79 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -909,4 +909,14 @@ Based on: ### Generated - [python v1.1.11] . ### Releases -- [PyPI v1.1.11] https://pypi.org/project/openrouter/1.1.11 - . \ No newline at end of file +- [PyPI v1.1.11] https://pypi.org/project/openrouter/1.1.11 - . + +## 2026-07-29 03:06:23 +### Changes +Based on: +- OpenAPI Doc +- Speakeasy CLI 1.787.0 (2.914.0) https://github.com/speakeasy-api/speakeasy +### Generated +- [python v1.1.12] . +### Releases +- [PyPI v1.1.12] https://pypi.org/project/openrouter/1.1.12 - . \ No newline at end of file diff --git a/docs/components/benchmarktype.mdx b/docs/components/benchmarktype.mdx new file mode 100644 index 00000000..2a473fdf --- /dev/null +++ b/docs/components/benchmarktype.mdx @@ -0,0 +1,22 @@ +--- +title: "BenchmarkType" +--- + +OpenRouter benchmark evaluation type. + +## Example Usage + +```python +from openrouter.components import BenchmarkType + +# Open enum: unrecognized values are captured as UnrecognizedStr +value: BenchmarkType = "gpqa_diamond" +``` + + +## Values + +This is an open enum. Unrecognized values will not fail type checks. + +- `"gpqa_diamond"` +- `"tau_bench_verified_airline"` diff --git a/docs/components/unifiedbenchmarksmetasource.mdx b/docs/components/unifiedbenchmarksmetasource.mdx index 1bcb8e01..5528a6a2 100644 --- a/docs/components/unifiedbenchmarksmetasource.mdx +++ b/docs/components/unifiedbenchmarksmetasource.mdx @@ -20,3 +20,4 @@ This is an open enum. Unrecognized values will not fail type checks. - `"artificial-analysis"` - `"design-arena"` +- `"openrouter"` diff --git a/docs/components/unifiedbenchmarksoritem.mdx b/docs/components/unifiedbenchmarksoritem.mdx new file mode 100644 index 00000000..99228c0b --- /dev/null +++ b/docs/components/unifiedbenchmarksoritem.mdx @@ -0,0 +1,17 @@ +--- +title: "UnifiedBenchmarksORItem" +--- + +## Fields + +| Field | Type | Required | Description | Example | +| ------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------ | +| `accuracy` | *float* | :heavy_check_mark: | Aggregate accuracy score from 0 to 1. Higher is better. | 0.72 | +| `accuracy_stddev` | *Nullable[float]* | :heavy_check_mark: | Standard deviation of run accuracy, or null for a single run. | 0.03 | +| `avg_cost_per_task` | *Nullable[float]* | :heavy_check_mark: | Average cost per task in USD, or null if unavailable. | 0.002 | +| `benchmark_type` | [components.BenchmarkType](../components/benchmarktype.mdx) | :heavy_check_mark: | OpenRouter benchmark evaluation type. | gpqa_diamond | +| `display_name` | *str* | :heavy_check_mark: | Human-readable model name. | GPT-4o | +| `last_run_timestamp` | *str* | :heavy_check_mark: | Timestamp of the most recent public benchmark run. | 2026-06-03T12:00:00Z | +| `model_permaslug` | *str* | :heavy_check_mark: | Stable OpenRouter model identifier. | openai/gpt-4o | +| `source` | [components.UnifiedBenchmarksORItemSource](../components/unifiedbenchmarksoritemsource.mdx) | :heavy_check_mark: | Benchmark source discriminator. | | +| `total_tasks` | *int* | :heavy_check_mark: | Total benchmark tasks across runs. | 300 | \ No newline at end of file diff --git a/docs/components/unifiedbenchmarksoritemsource.mdx b/docs/components/unifiedbenchmarksoritemsource.mdx new file mode 100644 index 00000000..558fb378 --- /dev/null +++ b/docs/components/unifiedbenchmarksoritemsource.mdx @@ -0,0 +1,17 @@ +--- +title: "UnifiedBenchmarksORItemSource" +--- + +Benchmark source discriminator. + +## Example Usage + +```python +from openrouter.components import UnifiedBenchmarksORItemSource +value: UnifiedBenchmarksORItemSource = "openrouter" +``` + + +## Values + +- `"openrouter"` diff --git a/docs/components/unifiedbenchmarksresponsedata.mdx b/docs/components/unifiedbenchmarksresponsedata.mdx index eb8045de..ff4c0671 100644 --- a/docs/components/unifiedbenchmarksresponsedata.mdx +++ b/docs/components/unifiedbenchmarksresponsedata.mdx @@ -16,3 +16,9 @@ value: components.UnifiedBenchmarksAAItem = /* values here */ value: components.UnifiedBenchmarksDAItem = /* values here */ ``` +### `components.UnifiedBenchmarksORItem` + +```python +value: components.UnifiedBenchmarksORItem = /* values here */ +``` + diff --git a/docs/operations/source.mdx b/docs/operations/source.mdx index 20b67f6c..85b44c9e 100644 --- a/docs/operations/source.mdx +++ b/docs/operations/source.mdx @@ -20,3 +20,4 @@ This is an open enum. Unrecognized values will not fail type checks. - `"artificial-analysis"` - `"design-arena"` +- `"openrouter"` diff --git a/docs/sdks/benchmarks/README.mdx b/docs/sdks/benchmarks/README.mdx index 072d78b5..da27ed83 100644 --- a/docs/sdks/benchmarks/README.mdx +++ b/docs/sdks/benchmarks/README.mdx @@ -13,7 +13,7 @@ Benchmarks endpoints ## get_benchmarks -Unified benchmark endpoint that aggregates scores from multiple benchmark sources (Artificial Analysis, Design Arena). Filter by source to reproduce the exact shapes from the legacy per-source endpoints, or use task_type to find models suited for specific workloads. Authenticate with any valid OpenRouter API key. Rate-limited to 30 requests/minute per key and 500 requests/day per account. +Unified benchmark endpoint that aggregates scores from multiple benchmark sources (Artificial Analysis, Design Arena, and OpenRouter's own tau-bench and GPQA evals). Filter by source to reproduce the exact shapes from the legacy per-source endpoints, or use task_type to find models suited for specific workloads. Authenticate with any valid OpenRouter API key. Rate-limited to 30 requests/minute per key and 500 requests/day per account. ### Example Usage diff --git a/pyproject.toml b/pyproject.toml index 9ad37b87..0ac27011 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "openrouter" -version = "1.1.11" +version = "1.1.12" description = "Official Python Client SDK for OpenRouter." authors = [{ name = "OpenRouter" },] readme = "README-PYPI.md" diff --git a/src/openrouter/_version.py b/src/openrouter/_version.py index 3d3b3319..c009a9e6 100644 --- a/src/openrouter/_version.py +++ b/src/openrouter/_version.py @@ -3,10 +3,10 @@ import importlib.metadata __title__: str = "openrouter" -__version__: str = "1.1.11" +__version__: str = "1.1.12" __openapi_doc_version__: str = "1.0.0" __gen_version__: str = "2.914.0" -__user_agent__: str = "speakeasy-sdk/python 1.1.11 2.914.0 1.0.0 openrouter" +__user_agent__: str = "speakeasy-sdk/python 1.1.12 2.914.0 1.0.0 openrouter" try: if __package__ is not None: diff --git a/src/openrouter/benchmarks.py b/src/openrouter/benchmarks.py index 1b503c89..5dcb0e04 100644 --- a/src/openrouter/benchmarks.py +++ b/src/openrouter/benchmarks.py @@ -30,7 +30,7 @@ def get_benchmarks( ) -> components.UnifiedBenchmarksResponse: r"""List Benchmarks - Unified benchmark endpoint that aggregates scores from multiple benchmark sources (Artificial Analysis, Design Arena). Filter by source to reproduce the exact shapes from the legacy per-source endpoints, or use task_type to find models suited for specific workloads. Authenticate with any valid OpenRouter API key. Rate-limited to 30 requests/minute per key and 500 requests/day per account. + Unified benchmark endpoint that aggregates scores from multiple benchmark sources (Artificial Analysis, Design Arena, and OpenRouter's own tau-bench and GPQA evals). Filter by source to reproduce the exact shapes from the legacy per-source endpoints, or use task_type to find models suited for specific workloads. Authenticate with any valid OpenRouter API key. Rate-limited to 30 requests/minute per key and 500 requests/day per account. :param http_referer: The app identifier should be your app's URL and is used as the primary identifier for rankings. This is used to track API usage per application. @@ -177,7 +177,7 @@ async def get_benchmarks_async( ) -> components.UnifiedBenchmarksResponse: r"""List Benchmarks - Unified benchmark endpoint that aggregates scores from multiple benchmark sources (Artificial Analysis, Design Arena). Filter by source to reproduce the exact shapes from the legacy per-source endpoints, or use task_type to find models suited for specific workloads. Authenticate with any valid OpenRouter API key. Rate-limited to 30 requests/minute per key and 500 requests/day per account. + Unified benchmark endpoint that aggregates scores from multiple benchmark sources (Artificial Analysis, Design Arena, and OpenRouter's own tau-bench and GPQA evals). Filter by source to reproduce the exact shapes from the legacy per-source endpoints, or use task_type to find models suited for specific workloads. Authenticate with any valid OpenRouter API key. Rate-limited to 30 requests/minute per key and 500 requests/day per account. :param http_referer: The app identifier should be your app's URL and is used as the primary identifier for rankings. This is used to track API usage per application. diff --git a/src/openrouter/components/__init__.py b/src/openrouter/components/__init__.py index 10eb82a8..2656d514 100644 --- a/src/openrouter/components/__init__.py +++ b/src/openrouter/components/__init__.py @@ -2731,6 +2731,12 @@ UnifiedBenchmarksMetaTypedDict, UnifiedBenchmarksMetaVersion, ) + from .unifiedbenchmarksoritem import ( + BenchmarkType, + UnifiedBenchmarksORItem, + UnifiedBenchmarksORItemSource, + UnifiedBenchmarksORItemTypedDict, + ) from .unifiedbenchmarksresponse import ( UnifiedBenchmarksResponse, UnifiedBenchmarksResponseData, @@ -3159,6 +3165,7 @@ "BashServerToolEnvironmentTypedDict", "BashServerToolType", "BashServerToolTypedDict", + "BenchmarkType", "Billable", "BooleanCapability", "BooleanCapabilityType", @@ -4897,6 +4904,9 @@ "UnifiedBenchmarksMetaSource", "UnifiedBenchmarksMetaTypedDict", "UnifiedBenchmarksMetaVersion", + "UnifiedBenchmarksORItem", + "UnifiedBenchmarksORItemSource", + "UnifiedBenchmarksORItemTypedDict", "UnifiedBenchmarksResponse", "UnifiedBenchmarksResponseData", "UnifiedBenchmarksResponseDataTypedDict", @@ -7058,6 +7068,10 @@ "UnifiedBenchmarksMetaSource": ".unifiedbenchmarksmeta", "UnifiedBenchmarksMetaTypedDict": ".unifiedbenchmarksmeta", "UnifiedBenchmarksMetaVersion": ".unifiedbenchmarksmeta", + "BenchmarkType": ".unifiedbenchmarksoritem", + "UnifiedBenchmarksORItem": ".unifiedbenchmarksoritem", + "UnifiedBenchmarksORItemSource": ".unifiedbenchmarksoritem", + "UnifiedBenchmarksORItemTypedDict": ".unifiedbenchmarksoritem", "UnifiedBenchmarksResponse": ".unifiedbenchmarksresponse", "UnifiedBenchmarksResponseData": ".unifiedbenchmarksresponse", "UnifiedBenchmarksResponseDataTypedDict": ".unifiedbenchmarksresponse", diff --git a/src/openrouter/components/unifiedbenchmarksmeta.py b/src/openrouter/components/unifiedbenchmarksmeta.py index 36d695e2..68d76b7b 100644 --- a/src/openrouter/components/unifiedbenchmarksmeta.py +++ b/src/openrouter/components/unifiedbenchmarksmeta.py @@ -11,6 +11,7 @@ Literal[ "artificial-analysis", "design-arena", + "openrouter", ], UnrecognizedStr, ] diff --git a/src/openrouter/components/unifiedbenchmarksoritem.py b/src/openrouter/components/unifiedbenchmarksoritem.py new file mode 100644 index 00000000..c58ba941 --- /dev/null +++ b/src/openrouter/components/unifiedbenchmarksoritem.py @@ -0,0 +1,85 @@ +"""Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.""" + +from __future__ import annotations +from openrouter.types import BaseModel, Nullable, UNSET_SENTINEL, UnrecognizedStr +from pydantic import model_serializer +from typing import Literal, Union +from typing_extensions import TypedDict + + +BenchmarkType = Union[ + Literal[ + "gpqa_diamond", + "tau_bench_verified_airline", + ], + UnrecognizedStr, +] +r"""OpenRouter benchmark evaluation type.""" + + +UnifiedBenchmarksORItemSource = Literal["openrouter",] +r"""Benchmark source discriminator.""" + + +class UnifiedBenchmarksORItemTypedDict(TypedDict): + accuracy: float + r"""Aggregate accuracy score from 0 to 1. Higher is better.""" + accuracy_stddev: Nullable[float] + r"""Standard deviation of run accuracy, or null for a single run.""" + avg_cost_per_task: Nullable[float] + r"""Average cost per task in USD, or null if unavailable.""" + benchmark_type: BenchmarkType + r"""OpenRouter benchmark evaluation type.""" + display_name: str + r"""Human-readable model name.""" + last_run_timestamp: str + r"""Timestamp of the most recent public benchmark run.""" + model_permaslug: str + r"""Stable OpenRouter model identifier.""" + source: UnifiedBenchmarksORItemSource + r"""Benchmark source discriminator.""" + total_tasks: int + r"""Total benchmark tasks across runs.""" + + +class UnifiedBenchmarksORItem(BaseModel): + accuracy: float + r"""Aggregate accuracy score from 0 to 1. Higher is better.""" + + accuracy_stddev: Nullable[float] + r"""Standard deviation of run accuracy, or null for a single run.""" + + avg_cost_per_task: Nullable[float] + r"""Average cost per task in USD, or null if unavailable.""" + + benchmark_type: BenchmarkType + r"""OpenRouter benchmark evaluation type.""" + + display_name: str + r"""Human-readable model name.""" + + last_run_timestamp: str + r"""Timestamp of the most recent public benchmark run.""" + + model_permaslug: str + r"""Stable OpenRouter model identifier.""" + + source: UnifiedBenchmarksORItemSource + r"""Benchmark source discriminator.""" + + total_tasks: int + r"""Total benchmark tasks across runs.""" + + @model_serializer(mode="wrap") + def serialize_model(self, handler): + serialized = handler(self) + m = {} + + for n, f in type(self).model_fields.items(): + k = f.alias or n + val = serialized.get(k, serialized.get(n)) + + if val != UNSET_SENTINEL: + m[k] = val + + return m diff --git a/src/openrouter/components/unifiedbenchmarksresponse.py b/src/openrouter/components/unifiedbenchmarksresponse.py index 7145a084..311cb86d 100644 --- a/src/openrouter/components/unifiedbenchmarksresponse.py +++ b/src/openrouter/components/unifiedbenchmarksresponse.py @@ -10,6 +10,10 @@ UnifiedBenchmarksDAItemTypedDict, ) from .unifiedbenchmarksmeta import UnifiedBenchmarksMeta, UnifiedBenchmarksMetaTypedDict +from .unifiedbenchmarksoritem import ( + UnifiedBenchmarksORItem, + UnifiedBenchmarksORItemTypedDict, +) from functools import partial from openrouter.types import BaseModel from openrouter.utils.unions import parse_open_union @@ -21,7 +25,11 @@ UnifiedBenchmarksResponseDataTypedDict = TypeAliasType( "UnifiedBenchmarksResponseDataTypedDict", - Union[UnifiedBenchmarksAAItemTypedDict, UnifiedBenchmarksDAItemTypedDict], + Union[ + UnifiedBenchmarksAAItemTypedDict, + UnifiedBenchmarksORItemTypedDict, + UnifiedBenchmarksDAItemTypedDict, + ], ) @@ -38,6 +46,7 @@ class UnknownUnifiedBenchmarksResponseData(BaseModel): _UNIFIED_BENCHMARKS_RESPONSE_DATA_VARIANTS: dict[str, Any] = { "artificial-analysis": UnifiedBenchmarksAAItem, "design-arena": UnifiedBenchmarksDAItem, + "openrouter": UnifiedBenchmarksORItem, } @@ -45,6 +54,7 @@ class UnknownUnifiedBenchmarksResponseData(BaseModel): Union[ UnifiedBenchmarksAAItem, UnifiedBenchmarksDAItem, + UnifiedBenchmarksORItem, UnknownUnifiedBenchmarksResponseData, ], BeforeValidator( diff --git a/src/openrouter/operations/getbenchmarks.py b/src/openrouter/operations/getbenchmarks.py index 493d52d3..62573095 100644 --- a/src/openrouter/operations/getbenchmarks.py +++ b/src/openrouter/operations/getbenchmarks.py @@ -77,6 +77,7 @@ def serialize_model(self, handler): Literal[ "artificial-analysis", "design-arena", + "openrouter", ], UnrecognizedStr, ] diff --git a/uv.lock b/uv.lock index 1062c7b3..66727e17 100644 --- a/uv.lock +++ b/uv.lock @@ -213,7 +213,7 @@ wheels = [ [[package]] name = "openrouter" -version = "1.1.11" +version = "1.1.12" source = { editable = "." } dependencies = [ { name = "httpcore" },