tests.txt

LLMConfigPrimitives
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  item 0 llm:active => "openai"
  member? "openai" llm:providers-all => true
  empty? (llm:provider-help "openai") => false
  empty? llm:config => false
  is-string? llm:list-models => true

LLMProvidersAll
  extensions [llm]
  length llm:providers-all => 8
  member? "openai" llm:providers-all => true
  member? "anthropic" llm:providers-all => true
  member? "gemini" llm:providers-all => true
  member? "ollama" llm:providers-all => true
  member? "openrouter" llm:providers-all => true
  member? "together" llm:providers-all => true
  member? "groq" llm:providers-all => true
  member? "typesafe" llm:providers-all => true

LLMProviderStatus
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  length llm:provider-status => 8

LLMListModels
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  is-string? llm:list-models => true
  empty? llm:list-models => false

LLMActiveConfig
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  item 0 llm:active => "openai"

LLMConfigSummary
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  is-string? llm:config => true
  empty? llm:config => false

LLMHistoryManagement
  extensions [llm]
  globals [history]
  O> llm:clear-history
  llm:history => []
  O> llm:set-history [["user" "hello"] ["assistant" "hi"]]
  O> set history llm:history
  length history => 2
  (item 0 (item 0 history)) => "user"
  (item 1 (item 0 history)) => "hello"
  (item 0 (item 1 history)) => "assistant"
  (item 1 (item 1 history)) => "hi"
  O> llm:clear-history
  llm:history => []

LLMSetModel
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:set-model "gpt-4o"
  item 1 llm:active => "gpt-4o"

LLMThinkingConfigPrimitives
  extensions [llm]
  O> llm:set-thinking true
  O> llm:set-reasoning-effort "high"
  O> llm:set-thinking-budget 4096
  O> llm:set-thinking false

LLMReasoningEffortValidation
  extensions [llm]
  O> llm:set-reasoning-effort "none"
  O> llm:set-reasoning-effort "low"
  O> llm:set-reasoning-effort "medium"
  O> llm:set-reasoning-effort "high"
  O> llm:set-reasoning-effort "xhigh"

LLMReasoningEffortIsProviderAware
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "groq"
  O> llm:set-reasoning-effort "low"
  O> llm:set-reasoning-effort "high"
  O> llm:set-reasoning-effort "xhigh" => ERROR Extension exception: Invalid reasoning effort: 'xhigh' for provider 'groq'. Must be one of: high, low, medium
  O> llm:set-reasoning-effort "none" => ERROR Extension exception: Invalid reasoning effort: 'none' for provider 'groq'. Must be one of: high, low, medium
  O> llm:set-provider "openai"
  O> llm:set-reasoning-effort "xhigh"
  O> llm:set-reasoning-effort "none"

LLMThinkingBudgetValidation
  extensions [llm]
  O> llm:set-thinking-budget 1024
  O> llm:set-thinking-budget 8192

LLMChat
  extensions [llm]
  globals [response]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> set response llm:chat "Say hello"
  is-string? response => true
  empty? response => false

LLMChatAsync
  extensions [llm]
  globals [promise result]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> set promise llm:chat-async "Say hi"
  is-anonymous-reporter? promise => true
  O> set result (runresult promise)
  is-string? result => true
  empty? result => false

LLMChatHistory
  extensions [llm]
  globals [r1 r2]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:clear-history
  O> set r1 llm:chat "My name is TestBot"
  O> set r2 llm:chat "What is my name?"
  length llm:history => 4
  (item 0 (item 0 llm:history)) => "user"
  (item 0 (item 1 llm:history)) => "assistant"

LLMChooseExactMatch
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  member? (llm:choose "Pick one" ["apple" "banana" "cherry"]) ["apple" "banana" "cherry"] => true

LLMChooseCaseInsensitive
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  (llm:choose "Pick one __TEST_RESPOND:APPLE" ["apple" "banana"]) => "apple"

LLMChooseEmptyListThrows
  extensions [llm]
  globals [err]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> carefully [let __x llm:choose "Pick" []] [set err error-message]
  is-string? err => true

LLMChooseNoMatchThrows
  extensions [llm]
  globals [err]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> carefully [let __x llm:choose "Pick one __TEST_RESPOND:zzzz" ["apple" "banana"]] [set err error-message]
  is-string? err => true

LLMChooseHistoryClean
  extensions [llm]
  globals [result]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:clear-history
  O> set result llm:choose "Pick a fruit" ["apple" "banana" "cherry"]
  length llm:history => 2
  (item 0 (item 0 llm:history)) => "user"
  (item 1 (item 0 llm:history)) => "Pick a fruit"
  (item 0 (item 1 llm:history)) => "assistant"
  member? (item 1 (item 1 llm:history)) ["apple" "banana" "cherry"] => true

LLMChooseHistoryUnchangedOnFailure
  extensions [llm]
  globals [err]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:clear-history
  O> llm:set-history [["user" "prior"] ["assistant" "context"]]
  O> carefully [let __x llm:choose "Pick __TEST_RESPOND:zzzz" ["apple" "banana"]] [set err error-message]
  length llm:history => 2
  (item 1 (item 0 llm:history)) => "prior"

LLMChooseThinkingModel
  extensions [llm]
  globals [result]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:clear-history
  O> set result llm:choose "Pick one __TEST_THINKING:apple" ["apple" "banana"]
  result => "apple"

LLMChatHistoryCorrectOnSuccess
  extensions [llm]
  globals [result]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:clear-history
  O> set result llm:chat "hello"
  length llm:history => 2
  (item 1 (item 0 llm:history)) => "hello"
  (item 0 (item 1 llm:history)) => "assistant"

LLMTemplateChat
  extensions [llm]
  globals [result]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> set result llm:chat-with-template "demos/simple-template.yaml" [["text" "hello world"]]
  is-string? result => true
  empty? result => false

LLMTemplateWithHistory
  extensions [llm]
  globals [msg1 template-result]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:clear-history
  O> set msg1 llm:chat "previous message"
  O> set template-result llm:chat-with-template "demos/simple-template.yaml" [["text" "with history"]]
  is-string? template-result => true
  empty? template-result => false
  length llm:history => 4

LLMChatWithThinking
  extensions [llm]
  globals [result]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:set-thinking true
  O> set result llm:chat-with-thinking "What is 2 + 2?"
  is-list? result => true
  length result => 2
  is-string? (item 0 result) => true
  is-string? (item 1 result) => true
  empty? (item 0 result) => false

LLMChatWithThinkingBudget
  extensions [llm]
  globals [result]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:set-thinking true
  O> llm:set-thinking-budget 2048
  O> set result llm:chat-with-thinking "What is 3 + 5?"
  is-list? result => true
  length result => 2
  is-string? (item 0 result) => true
  empty? (item 0 result) => false

LLMMultiAgentChat
  extensions [llm]
  turtles-own [agent-result]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> create-turtles 2
  O> ask turtle 0 [ set agent-result llm:chat "Say one" ]
  O> ask turtle 1 [ set agent-result llm:chat "Say two" ]
  is-string? [agent-result] of turtle 0 => true
  is-string? [agent-result] of turtle 1 => true
  empty? [agent-result] of turtle 0 => false
  empty? [agent-result] of turtle 1 => false

LLMOpenRouterProvider
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openrouter"
  item 0 llm:active => "openrouter"
  item 1 llm:active => "openai/gpt-oss-20b"
  empty? (llm:provider-help "openrouter") => false
  O> llm:set-model "openai/gpt-4o"
  item 1 llm:active => "openai/gpt-4o"

LLMTogetherProvider
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "together"
  item 0 llm:active => "together"
  item 1 llm:active => "Qwen/Qwen3.8-Flash"
  empty? (llm:provider-help "together") => false
  O> llm:set-model "deepseek-ai/DeepSeek-R1"
  item 1 llm:active => "deepseek-ai/DeepSeek-R1"

LLMGroqProvider
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "groq"
  item 0 llm:active => "groq"
  item 1 llm:active => "openai/gpt-oss-20b"
  empty? (llm:provider-help "groq") => false
  O> llm:set-model "openai/gpt-oss-120b"
  item 1 llm:active => "openai/gpt-oss-120b"

LLMGroqChat
  extensions [llm]
  globals [response]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "groq"
  O> llm:clear-history
  O> set response llm:chat "Say hello"
  is-string? response => true
  empty? response => false
  length llm:history => 2

LLMConfigLoading
  extensions [llm]
  O> llm:load-config "demos/config"
  item 0 llm:active => "openai"
  member? "openai" llm:providers => true

LLMLoadConfigRejectsUnknownProvider
  extensions [llm]
  globals [err]
  O> carefully [llm:load-config "demos/test-config-bad-provider"] [set err error-message]
  is-string? err => true
  (position "opanai" err) != false => true

LLMChatHistoryUnchangedOnFailure
  extensions [llm]
  globals [err]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:clear-history
  O> llm:set-history [["user" "prior"] ["assistant" "context"]]
  O> carefully [let __x llm:chat "__TEST_FAIL"] [set err error-message]
  is-string? err => true
  length llm:history => 2
  (item 1 (item 0 llm:history)) => "prior"

LLMChatWithThinkingHistoryUnchangedOnFailure
  extensions [llm]
  globals [err]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:clear-history
  O> llm:set-history [["user" "prior"] ["assistant" "context"]]
  O> carefully [let __x llm:chat-with-thinking "__TEST_FAIL"] [set err error-message]
  is-string? err => true
  length llm:history => 2
  (item 1 (item 0 llm:history)) => "prior"

LLMChatAsyncOverlappingPairsAtomic
  extensions [llm]
  globals [p1 p2 r1 r2 h]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:clear-history
  O> set p1 llm:chat-async "__TEST_DELAY:300:slow"
  O> set p2 llm:chat-async "fast"
  O> set r1 (runresult p1)
  O> set r2 (runresult p2)
  r1 => "stub:slow"
  r2 => "stub:fast"
  O> set h llm:history
  length h => 4
  (item 0 (item 0 h)) => "user"
  (item 0 (item 1 h)) => "assistant"
  (item 0 (item 2 h)) => "user"
  (item 0 (item 3 h)) => "assistant"
  member? (item 1 (item 1 h)) ["stub:slow" "stub:fast"] => true
  member? (item 1 (item 3 h)) ["stub:slow" "stub:fast"] => true

LLMMixedSyncAsyncHistoryAtomic
  extensions [llm]
  globals [p1 r1 r2 h]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:clear-history
  O> set p1 llm:chat-async "__TEST_DELAY:300:slow"
  O> set r1 llm:chat "quick"
  O> set r2 (runresult p1)
  O> set h llm:history
  length h => 4
  (item 0 (item 0 h)) => "user"
  (item 0 (item 1 h)) => "assistant"
  (item 0 (item 2 h)) => "user"
  (item 0 (item 3 h)) => "assistant"

LLMCompileErrorValid
  extensions [llm]
  llm:compile-error "fd 1" => ""
  llm:compile-error "rt 90 fd 1" => ""
  llm:compile-error "" => ""
  llm:compile-error "   " => ""
  llm:compile-error "ifelse xcor > 0 [ fd 1 ] [ bk 1 ]" => ""
  llm:compile-error "rt (random 90) * 0.5 + (random 45) * 0.5" => ""

LLMCompileErrorInvalid
  extensions [llm]
  (llm:compile-error "frobnicate") != "" => true
  (llm:compile-error "fd") != "" => true
  (llm:compile-error "if xcor > 0 [fd 1] [bk 1]") != "" => true
  (llm:compile-error "ifelse xcor > 0 [ fd 1 ]") != "" => true
  (llm:compile-error "rt undefined-var-xyz") != "" => true
  is-string? (llm:compile-error "frobnicate") => true

LLMCompileErrorTurtleContext
  extensions [llm]
  llm:compile-error "set heading 90" => ""
  llm:compile-error "set color red fd 1" => ""


LLMCompileErrorLEARCorpus
  extensions [llm]
  llm:compile-error "ifelse random 2 < 1 [lt 10 fd 1] [ifelse random 2 < 1 [rt 10 fd 1] [fd 1] ]" => ""
  llm:compile-error "ifelse random 2 = 0 [ lt random 20 fd 1 ] [ rt random 20 fd 1 ]" => ""
  llm:compile-error "ifelse random 2 = 0 [fd 1] [rt 90 fd 1]" => ""
  llm:compile-error "ifelse random 2 = 0 [lt random 10 fd 1] [rt random 10 fd 1]" => ""
  llm:compile-error "ifelse random 2 = 0 [lt random 20 fd 1] [rt random 10 fd 1]" => ""
  llm:compile-error "ifelse random 2 = 0 [lt random 30 fd 1] [rt random 30 fd 1]" => ""
  llm:compile-error "ifelse random 2 = 0 [rt random 30 fd 1] [lt random 30 fd 1]" => ""
  llm:compile-error "ifelse random 2 = 0 [ifelse random 2 = 0 [lt 10 fd 1] [rt 10 fd 1]] [ifelse random 2 = 0 [lt random 10 fd 1] [rt random 10 fd 1]]" => ""
  llm:compile-error "ifelse random 2 = 0 [lt random 10 fd 1] [ifelse random 2 = 0 [rt random 10 fd 1] [fd 1] ]" => ""
  llm:compile-error "ifelse random 2 = 0 [lt random 10 fd 1] [ifelse random 2 = 1 [rt random 10 fd 1] [fd 1] ]" => ""
  llm:compile-error "ifelse random 2 = 0 [lt random 20 fd 1] [rt random 20 fd 1]" => ""
  llm:compile-error "ifelse random 2 = 0 [lt random-float 10 fd 1] [rt random-float 10 fd 1]" => ""
  llm:compile-error "ifelse random 2 = 0 [ fd 1 ] [ ifelse random 2 = 1 [ rt 10 fd 1 ] [ lt 10 fd 1 ] ]" => ""
  llm:compile-error "ifelse random-float 1 < 0.4 [lt random 10 fd 1] [ifelse random-float 1 < 0.3 [rt random 10 fd 1] [fd 1] ]" => ""

LLMCompileErrorBannedPrimitives
  extensions [llm]
  (llm:compile-error "fd 1" ["die"]) => ""
  (llm:compile-error "fd 1" []) => ""
  (llm:compile-error "die" ["die"]) => "Disallowed primitive(s) used: die"
  (llm:compile-error "fd 1 die" ["die" "clear-all"]) => "Disallowed primitive(s) used: die"
  (llm:compile-error "DIE" ["die"]) => "Disallowed primitive(s) used: die"
  (llm:compile-error "die" ["DIE"]) => "Disallowed primitive(s) used: die"

LLMCompileErrorBannedNoFalsePositives
  extensions [llm]
  (llm:compile-error "fd 1 ; die is banned" ["die"]) => ""
  (llm:compile-error "let diehard 1 fd diehard" ["die"]) => ""
  (llm:compile-error "print \"die\"" ["die"]) => ""

LLMCompileErrorSyntaxBeforeBanned
  extensions [llm]
  (llm:compile-error "frobnicate die" ["die"]) != "" => true
  member? "Disallowed" (llm:compile-error "frobnicate die" ["die"]) => false

LLMCompileErrorMultiLineLocals
  extensions [llm]
  llm:compile-error "let x 5\nfd x" => ""
  llm:compile-error "let x 5\nlet y x * 2\nrt y\nfd 1" => ""
  llm:compile-error "let x 5\n\n  rt x\n  fd 1\n" => ""
  llm:compile-error "let d 3\nifelse d > 2\n  [ fd d ]\n  [ rt 90 fd 1 ]" => ""
  llm:compile-error "  \n  fd 1  \n  " => ""
  llm:compile-error "let x 1 let y 2 let z 3 fd x + y + z" => ""
  (llm:compile-error "let x 5\nfd undefined-thing") != "" => true
  (llm:compile-error "fd x\nlet x 5") != "" => true
  (llm:compile-error "let x 5\nlet x 6\nfd x") != "" => true
  (llm:compile-error "let x 5\nlet x 6\nfd x") => "There is already a local variable here called X (offset 43)"

LLMCompileErrorTurtleOwnVariables
  extensions [llm]
  llm:compile-error "fd xcor" => ""
  llm:compile-error "set heading heading + 10 fd 1" => ""
  llm:compile-error "ifelse pcolor = green [ fd 1 ] [ rt 90 ]" => ""
  llm:compile-error "let n count turtles in-radius 3\nfd n" => ""
  (llm:compile-error "fd not-a-turtle-variable") != "" => true

LLMCompileErrorComments
  extensions [llm]
  llm:compile-error "; only a comment" => ""
  llm:compile-error "fd 1 ; move forward" => ""
  llm:compile-error "; leading comment\nfd 1\n; trailing comment" => ""
  llm:compile-error "let x 5 ; set speed\nfd x" => ""

LLMCompileErrorNestedAndBlocks
  extensions [llm]
  llm:compile-error "ask other turtles in-radius 2 [ rt 180 ]" => ""
  llm:compile-error "repeat 3 [ fd 1 rt 120 ]" => ""
  llm:compile-error "if any? other turtles [ fd 1 ]" => ""
  llm:compile-error "(ifelse xcor > 5 [ rt 90 ] xcor < -5 [ lt 90 ] [ fd 1 ])" => ""
  llm:compile-error "foreach [1 2 3] [ n -> fd n ]" => ""
  (llm:compile-error "repeat 3 [ fd 1 rt 120") != "" => true
  (llm:compile-error "ask turtles [ fd 1 ] ]") != "" => true

LLMCompileErrorBannedMultiLine
  extensions [llm]
  (llm:compile-error "let x 5\nfd x\ndie" ["die"]) => "Disallowed primitive(s) used: die"
  (llm:compile-error "fd 1\n; die here is a comment\nrt 90" ["die"]) => ""
  (llm:compile-error "ask turtles [ die ]" ["die"]) => "Disallowed primitive(s) used: die"
  (llm:compile-error "fd 1\nrt 90" ["die" "clear-all" "hatch"]) => ""
  (llm:compile-error "let diehard 1\nfd diehard" ["die"]) => ""

LLMCompileErrorBannedMultipleFound
  extensions [llm]
  (llm:compile-error "ask turtles [ die ]" ["die" "ask"]) => "Disallowed primitive(s) used: ask, die"
  (llm:compile-error "die die die" ["die"]) => "Disallowed primitive(s) used: die"

LLMCompileErrorBannedEdgeCases
  extensions [llm]
  (llm:compile-error "" ["die"]) => ""
  (llm:compile-error "fd 1" ["  "]) => ""
  (llm:compile-error "fd 1" ["fd"]) => "Disallowed primitive(s) used: fd"
  (llm:compile-error "fd 1" ["  die  "]) => ""
  (llm:compile-error "die" ["  die  "]) => "Disallowed primitive(s) used: die"

LLMCompileErrorBannedListValidation
  extensions [llm]
  (llm:compile-error "die" [1 2]) => ERROR Extension exception: llm:compile-error expects a list of primitive names as strings, but got: 1, 2
  (llm:compile-error "fd 1" ["die" 5]) => ERROR Extension exception: llm:compile-error expects a list of primitive names as strings, but got: 5

LLMGetLookup
  extensions [llm]
  llm:get [["name" "Alice"] ["age" 30]] "name" => "Alice"
  llm:get [["name" "Alice"] ["age" 30]] "age" => 30
  llm:get [["flag" true]] "flag" => true
  llm:get [["skills" ["coding" "design"]]] "skills" => ["coding" "design"]
  llm:get (llm:get [["addr" [["city" "Chicago"]]]] "addr") "city" => "Chicago"
  llm:get [["name" "Alice"]] "missing" => ERROR Extension exception: llm:get: key "missing" not found. Available keys: name
  llm:get [] "any" => ERROR Extension exception: llm:get: key "any" not found. The list is empty.
  llm:get ["a" "b"] "a" => ERROR Extension exception: llm:get: key "a" not found. Available keys: (none - the list is not a list of [key value] pairs)

LLMChatWithSchemaParsesJson
  extensions [llm]
  globals [result]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:clear-history
  O> set result llm:chat-with-schema "__TEST_RESPOND:{\"action\":\"explore\",\"confidence\":0.8}" "{\"type\":\"object\",\"properties\":{\"action\":{\"type\":\"string\"},\"confidence\":{\"type\":\"number\"}}}"
  llm:get result "action" => "explore"
  llm:get result "confidence" => 0.8
  is-list? result => true
  O> llm:clear-history

LLMChatWithSchemaNested
  extensions [llm]
  globals [result]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:clear-history
  O> set result llm:chat-with-schema "__TEST_RESPOND:{\"plan\":[\"a\",\"b\"],\"meta\":{\"n\":2}}" "{\"type\":\"object\",\"properties\":{\"plan\":{\"type\":\"array\",\"items\":{\"type\":\"string\"}},\"meta\":{\"type\":\"object\",\"properties\":{\"n\":{\"type\":\"number\"}}}}}"
  llm:get result "plan" => ["a" "b"]
  llm:get (llm:get result "meta") "n" => 2
  O> llm:clear-history

LLMChatWithSchemaRejectsBadSchema
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  llm:chat-with-schema "hello" "not json at all" => ERROR Extension exception: llm:chat-with-schema: Schema is not valid JSON: expected null got "n" at index 0
  llm:chat-with-schema "hello" "" => ERROR Extension exception: llm:chat-with-schema: Schema cannot be empty. Provide a JSON Schema object, e.g. {"type":"object","properties":{"action":{"type":"string"}}}

LLMChatWithSchemaRejectsNonObjectTopLevel
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  llm:chat-with-schema "hi" "{\"type\":\"string\"}" => ERROR Extension exception: llm:chat-with-schema: schema must have type 'object' at the top level, but got 'string'. llm:chat-with-schema reports a list of [key value] pairs, so the reply has to be a JSON object. Wrap it, e.g. {"type":"object","properties":{"value":{"type":"string"}}}
  llm:chat-with-schema "hi" "{\"type\":\"array\",\"items\":{\"type\":\"string\"}}" => ERROR Extension exception: llm:chat-with-schema: schema must have type 'object' at the top level, but got 'array'. llm:chat-with-schema reports a list of [key value] pairs, so the reply has to be a JSON object. Wrap it, e.g. {"type":"object","properties":{"value":{"type":"string"}}}

LLMChatWithSchemaRejectsNonObjectReply
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:clear-history
  llm:chat-with-schema "__TEST_RESPOND:\"just a string\"" "{\"type\":\"object\",\"properties\":{\"a\":{\"type\":\"string\"}}}" => ERROR Extension exception: llm:chat-with-schema: expected a JSON object in the reply, but got a string. Response text: "just a string"
  llm:history => []
  O> llm:clear-history

LLMChatWithSchemaHistory
  extensions [llm]
  globals [result]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:clear-history
  O> set result llm:chat-with-schema "__TEST_RESPOND:{\"a\":1}" "{\"type\":\"object\",\"properties\":{\"a\":{\"type\":\"number\"}}}"
  length llm:history => 2
  item 0 (item 0 llm:history) => "user"
  item 0 (item 1 llm:history) => "assistant"
  O> llm:clear-history

LLMChatWithSchemaHistoryUnchangedOnFailure
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:clear-history
  O> carefully [ let r llm:chat-with-schema "__TEST_FAIL" "{\"type\":\"object\",\"properties\":{\"a\":{\"type\":\"number\"}}}" ] [ ]
  llm:history => []
  O> llm:clear-history

LLMChatWithSchemaRejectsNonJsonReply
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:clear-history
  llm:chat-with-schema "__TEST_RESPOND:sorry I cannot" "{\"type\":\"object\",\"properties\":{\"a\":{\"type\":\"number\"}}}" => ERROR Extension exception: llm:chat-with-schema: Response was not valid JSON: expected json value got "s" at index 0. Response text: sorry I cannot
  llm:history => []
  O> llm:clear-history

LLMChatJsonReturnsRawString
  extensions [llm]
  globals [result]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:clear-history
  O> set result llm:chat-json "__TEST_RESPOND:{\"a\":1}"
  is-string? result => true
  result => "{\"a\":1}"
  length llm:history => 2
  O> llm:clear-history

LLMChatJsonRejectsNonJsonReply
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:clear-history
  llm:chat-json "__TEST_RESPOND:Sure! Here is the JSON you asked for." => ERROR Extension exception: llm:chat-json: Response was not valid JSON: expected json value got "S" at index 0. Response text: Sure! Here is the JSON you asked for.
  llm:history => []
  O> llm:clear-history

LLMChatJsonAcceptsNonObjectJson
  extensions [llm]
  globals [result]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:clear-history
  O> set result llm:chat-json "__TEST_RESPOND:[1,2,3]"
  result => "[1,2,3]"
  length llm:history => 2
  O> llm:clear-history

LLMChatJsonHistoryUnchangedOnFailure
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:clear-history
  O> carefully [ let r llm:chat-json "__TEST_FAIL" ] [ ]
  llm:history => []
  O> llm:clear-history

LLMChooseStructured
  extensions [llm]
  globals [choice]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:clear-history
  O> set choice llm:choose "Pick a direction" ["north" "south" "east" "west"]
  member? choice ["north" "south" "east" "west"] => true
  length llm:history => 2
  O> llm:clear-history

LLMChooseStructuredEmptyList
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  llm:choose "Pick" [] => ERROR Extension exception: Choice list cannot be empty

LLMChooseSendsEnumConstraint
  extensions [llm]
  globals [choice]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:clear-history
  O> set choice llm:choose "__TEST_ECHO_FORMAT_ENUM" ["north" "south"]
  choice => "south"
  length llm:history => 2
  item 1 (item 1 llm:history) => "south"
  O> llm:clear-history
  O> set choice llm:choose "__TEST_ECHO_FORMAT_ENUM" ["alpha" "beta" "gamma"]
  choice => "gamma"
  O> llm:clear-history

LLMChatJsonSendsJsonObjectFormat
  extensions [llm]
  globals [result]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:clear-history
  O> set result llm:chat-json "__TEST_ECHO_FORMAT"
  result => "{\"format\":\"json_object\"}"
  O> llm:clear-history

LLMChatWithSchemaSendsSchemaFormat
  extensions [llm]
  globals [result]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:clear-history
  O> set result llm:chat-with-schema "__TEST_ECHO_FORMAT" "{\"type\":\"object\",\"properties\":{\"a\":{\"type\":\"string\"}}}"
  llm:get result "format" => "json_schema"
  llm:get result "topLevelType" => "object"
  llm:get result "name" => "netlogo_schema"
  O> llm:clear-history

LLMChoosePromptMatchesEnumSchema
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:clear-history
  llm:choose "__TEST_ECHO_PROMPT_TAIL" ["north" "south"] => ERROR Extension exception: llm:choose: response 'Your choice as {"choice": "<option>"}:' did not match any choice. Choices: north, south
  llm:history => []
  O> llm:clear-history

LLMChooseDegradesWhenFormatIgnored
  extensions [llm]
  globals [choice]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:clear-history
  O> set choice llm:choose "__TEST_IGNORE_FORMAT Pick a direction" ["north" "south" "east" "west"]
  choice => "north"
  length llm:history => 2
  item 1 (item 1 llm:history) => "north"
  O> llm:clear-history

LLMChooseDegradesCaseInsensitively
  extensions [llm]
  globals [choice]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:clear-history
  O> set choice llm:choose "pick __TEST_IGNORE_FORMAT __TEST_RAW_CHOICE:NORTH" ["north" "south"]
  choice => "north"
  length llm:history => 2
  O> llm:clear-history

LLMChooseDegradesOnMalformedConstrainedJson
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:clear-history
  llm:choose "__TEST_IGNORE_FORMAT __TEST_RAW_CHOICE:{\"choice\": pick" ["north" "south"] => ERROR Extension exception: llm:choose: response '{"choice": pick' did not match any choice. Choices: north, south
  llm:history => []
  O> llm:clear-history

LLMChooseUnmatchedAfterDegradationLeavesHistoryClean
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:clear-history
  llm:choose "__TEST_IGNORE_FORMAT __TEST_RAW_CHOICE:up" ["north" "south"] => ERROR Extension exception: llm:choose: response 'up' did not match any choice. Choices: north, south
  llm:history => []
  O> llm:clear-history

LLMUsageStartsAtZero
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> clear-all
  llm:get llm:usage "input-tokens" => 0
  llm:get llm:usage "output-tokens" => 0
  llm:get llm:usage "total-tokens" => 0
  llm:get llm:usage "calls" => 0
  llm:get llm:usage "cost" => ""
  llm:get llm:usage-total "calls" => 0

LLMUsageAccumulatesAcrossCalls
  extensions [llm]
  globals [r]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> clear-all
  O> set r llm:chat "__TEST_USAGE:10,5 hello"
  O> set r llm:chat "__TEST_USAGE:20,7,3 again"
  llm:get llm:usage "input-tokens" => 30
  llm:get llm:usage "output-tokens" => 12
  llm:get llm:usage "total-tokens" => 42
  llm:get llm:usage "reasoning-tokens" => 3
  llm:get llm:usage "calls" => 2
  O> clear-all

LLMUsageIsPerAgent
  extensions [llm]
  globals [r]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> clear-all
  O> crt 2
  O> ask turtles [ set r llm:chat (word "__TEST_USAGE:" (10 * (who + 1)) ",1 hi") ]
  [llm:get llm:usage "input-tokens"] of turtle 0 => 10
  [llm:get llm:usage "input-tokens"] of turtle 1 => 20
  llm:get llm:usage "input-tokens" => 0
  llm:get llm:usage-total "input-tokens" => 30
  llm:get llm:usage-total "output-tokens" => 2
  llm:get llm:usage-total "calls" => 2
  O> clear-all

LLMUsageResetsOnClearAll
  extensions [llm]
  globals [r]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> clear-all
  O> set r llm:chat "__TEST_USAGE:10,5 hi"
  llm:get llm:usage-total "calls" => 1
  O> clear-all
  llm:get llm:usage "calls" => 0
  llm:get llm:usage-total "calls" => 0

LLMUsageRecordsAsyncCalls
  extensions [llm]
  globals [p r]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> clear-all
  O> set p llm:chat-async "__TEST_USAGE:4,3 hi"
  O> set r (runresult p)
  llm:get llm:usage "total-tokens" => 7
  llm:get llm:usage "calls" => 1
  O> clear-all

LLMUsageRecordsEveryPrimitive
  extensions [llm]
  globals [r]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> clear-all
  O> set r llm:choose "__TEST_USAGE:1,1 pick" ["north" "south"]
  O> set r llm:chat-with-schema "__TEST_USAGE:2,1" "{\"type\":\"object\",\"properties\":{\"a\":{\"type\":\"string\"}}}"
  O> set r llm:chat-json "__TEST_USAGE:3,1"
  O> set r llm:chat-with-thinking "__TEST_USAGE:4,1 think"
  llm:get llm:usage "input-tokens" => 10
  llm:get llm:usage "calls" => 4
  O> clear-all

LLMUsageCountsRejectedReply
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> clear-all
  O> carefully [ let r llm:chat-with-schema "__TEST_USAGE:9,2 __TEST_RESPOND:not-json" "{\"type\":\"object\",\"properties\":{\"a\":{\"type\":\"string\"}}}" ] [ ]
  llm:history => []
  llm:get llm:usage "input-tokens" => 9
  llm:get llm:usage "calls" => 1
  O> clear-all

LLMUsageUnchangedWhenCallFails
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> clear-all
  O> carefully [ let r llm:chat "__TEST_USAGE:9,2 __TEST_FAIL" ] [ ]
  llm:get llm:usage "calls" => 0
  llm:get llm:usage "input-tokens" => 0
  O> clear-all

LLMProfileDefaultsToGlobal
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> clear-all
  llm:profile => "default"
  llm:profiles => []

LLMLoadProfileRejectsBadProvider
  extensions [llm]
  globals [err]
  O> carefully [llm:load-profile "bad" "demos/test-config-bad-provider"] [set err error-message]
  member? "Unknown provider" err => true
  llm:profiles => []

LLMLoadProfileRejectsMissingFile
  extensions [llm]
  globals [err]
  O> carefully [llm:load-profile "nope" "demos/no-such-file"] [set err error-message]
  member? "Failed to load" err => true
  llm:profiles => []

LLMLoadProfileRejectsReservedName
  extensions [llm]
  O> llm:load-profile "default" "demos/test-profile-a" => ERROR Extension exception: llm:load-profile: 'default' is reserved for the global configuration. Choose another name.

LLMUseProfileUnknownThrows
  extensions [llm]
  O> llm:load-profile "a" "demos/test-profile-a"
  O> llm:use-profile "zzz" => ERROR Extension exception: llm:use-profile: no profile named 'zzz'. Loaded profiles: a

LLMUseProfileRoutesToItsModel
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:set-model "model-default"
  O> clear-all
  O> llm:load-profile "a" "demos/test-profile-a"
  O> llm:load-profile "b" "demos/test-profile-b"
  O> crt 2
  O> ask turtle 0 [ llm:use-profile "a" ]
  O> ask turtle 1 [ llm:use-profile "b" ]
  [llm:chat "__TEST_ECHO_MODEL"] of turtle 0 => "model-a"
  [llm:chat "__TEST_ECHO_MODEL"] of turtle 1 => "model-b"
  llm:chat "__TEST_ECHO_MODEL" => "model-default"
  [llm:profile] of turtle 0 => "a"
  [llm:profile] of turtle 1 => "b"
  llm:profile => "default"
  llm:profiles => ["a" "b"]
  O> clear-all

LLMActiveAndConfigReportCallingAgentProfile
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:set-model "model-default"
  O> clear-all
  O> llm:load-profile "a" "demos/test-profile-a"
  O> crt 1
  O> ask turtle 0 [ llm:use-profile "a" ]
  [llm:active] of turtle 0 => ["openai" "model-a"]
  llm:active => ["openai" "model-default"]
  member? "model-a" [llm:config] of turtle 0 => true
  member? "model-a" llm:config => false
  O> clear-all

LLMUseProfileDefaultReturnsToGlobal
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:set-model "model-default"
  O> clear-all
  O> llm:load-profile "a" "demos/test-profile-a"
  O> crt 1
  O> ask turtle 0 [ llm:use-profile "a" ]
  [llm:chat "__TEST_ECHO_MODEL"] of turtle 0 => "model-a"
  O> ask turtle 0 [ llm:use-profile "default" ]
  [llm:chat "__TEST_ECHO_MODEL"] of turtle 0 => "model-default"
  [llm:profile] of turtle 0 => "default"
  O> clear-all

LLMProfileAsyncKeepsProviderAfterReassignment
  extensions [llm]
  globals [p r]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:set-model "model-default"
  O> clear-all
  O> llm:load-profile "a" "demos/test-profile-a"
  O> llm:load-profile "b" "demos/test-profile-b"
  O> crt 1
  O> ask turtle 0 [ llm:use-profile "a" ]
  O> ask turtle 0 [ set p llm:chat-async "__TEST_DELAY:300:__TEST_ECHO_MODEL" ]
  O> ask turtle 0 [ llm:use-profile "b" ]
  O> set r (runresult p)
  r => "model-a"
  [llm:chat "__TEST_ECHO_MODEL"] of turtle 0 => "model-b"
  O> clear-all

LLMSetModelLeavesProfilesAlone
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:set-model "model-default"
  O> clear-all
  O> llm:load-profile "a" "demos/test-profile-a"
  O> crt 1
  O> ask turtle 0 [ llm:use-profile "a" ]
  O> llm:set-model "model-changed"
  [llm:chat "__TEST_ECHO_MODEL"] of turtle 0 => "model-a"
  llm:chat "__TEST_ECHO_MODEL" => "model-changed"
  O> clear-all

LLMReloadProfileReplacesIt
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> clear-all
  O> llm:load-profile "a" "demos/test-profile-a"
  O> crt 1
  O> ask turtle 0 [ llm:use-profile "a" ]
  [llm:chat "__TEST_ECHO_MODEL"] of turtle 0 => "model-a"
  O> llm:load-profile "a" "demos/test-profile-b"
  [llm:chat "__TEST_ECHO_MODEL"] of turtle 0 => "model-b"
  llm:profiles => ["a"]
  O> clear-all

LLMFailedReloadKeepsOldProfile
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> clear-all
  O> llm:load-profile "a" "demos/test-profile-a"
  O> crt 1
  O> ask turtle 0 [ llm:use-profile "a" ]
  O> carefully [llm:load-profile "a" "demos/test-config-bad-provider"] [ ]
  [llm:chat "__TEST_ECHO_MODEL"] of turtle 0 => "model-a"
  O> clear-all

LLMClearAllResetsAssignmentsKeepsProfiles
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> clear-all
  O> llm:load-profile "a" "demos/test-profile-a"
  O> llm:use-profile "a"
  llm:profile => "a"
  O> clear-all
  llm:profile => "default"
  llm:profiles => ["a"]
  O> clear-all

LLMProfileHistoryAndUsageStayPerAgent
  extensions [llm]
  globals [r]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> clear-all
  O> llm:load-profile "a" "demos/test-profile-a"
  O> crt 2
  O> ask turtle 0 [ llm:use-profile "a" ]
  O> ask turtles [ set r llm:chat "__TEST_USAGE:5,1 hi" ]
  [length llm:history] of turtle 0 => 2
  [length llm:history] of turtle 1 => 2
  [llm:get llm:usage "input-tokens"] of turtle 0 => 5
  [llm:get llm:usage "input-tokens"] of turtle 1 => 5
  llm:get llm:usage-total "calls" => 2
  O> clear-all

LLMProfileWithoutModelUsesProviderDefault
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> clear-all
  O> llm:load-profile "claude" "demos/test-profile-anthropic"
  O> crt 1
  O> ask turtle 0 [ llm:use-profile "claude" ]
  [llm:active] of turtle 0 => ["anthropic" "claude-haiku-5-5"]
  O> clear-all

LLMProfileUsesItsOwnWaitBudget
  extensions [llm]
  globals [err r]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> clear-all
  O> llm:load-profile "slow" "demos/test-profile-slow"
  O> crt 1
  O> ask turtle 0 [ llm:use-profile "slow" ]
  O> ask turtle 0 [ carefully [ set r llm:chat "__TEST_DELAY:1500:hi" ] [ set err error-message ] ]
  member? "timed out" err => true
  O> set r llm:chat "__TEST_DELAY:1500:hi"
  r => "stub:hi"
  O> clear-all

LLMAsyncKeepsTheBudgetItStartedWith
  extensions [llm]
  globals [p r]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> clear-all
  O> llm:load-profile "a" "demos/test-profile-a"
  O> crt 1
  O> ask turtle 0 [ llm:use-profile "a" ]
  O> ask turtle 0 [ set p llm:chat-async "__TEST_DELAY:1500:hi" ]
  O> llm:load-config "demos/test-config-fast-timeout"
  O> set r (runresult p)
  r => "stub:hi"
  O> clear-all

LLMChatAllReturnsTriples
  extensions [llm]
  globals [r]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> clear-all
  O> crt 3
  O> set r llm:chat-all turtles [ -> "hi" ]
  length r => 3
  map [ x -> length x ] r => [3 3 3]
  sort map [ x -> [who] of item 0 x ] r => [0 1 2]
  map [ x -> item 1 x ] r => ["stub:hi" "stub:hi" "stub:hi"]
  map [ x -> item 2 x ] r => [true true true]
  O> clear-all

LLMChatAllPromptRunsAsEachAgent
  extensions [llm]
  globals [r]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> clear-all
  O> crt 3
  O> set r llm:chat-all turtles [ -> (word "id " who) ]
  reduce and map [ x -> item 1 x = (word "stub:id " [who] of item 0 x) ] r => true
  O> clear-all

LLMChatAllCommitsHistoryPerAgent
  extensions [llm]
  globals [r]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> clear-all
  O> crt 2
  O> set r llm:chat-all turtles [ -> (word "id " who) ]
  [length llm:history] of turtle 0 => 2
  [item 1 (item 0 llm:history)] of turtle 0 => "id 0"
  [item 1 (item 1 llm:history)] of turtle 1 => "stub:id 1"
  llm:history => []
  O> clear-all

LLMChatAllFailureIsolated
  extensions [llm]
  globals [r bad good]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> clear-all
  O> crt 3
  O> set r llm:chat-all turtles [ -> ifelse-value who = 1 [ "__TEST_FAIL" ] [ "ok" ] ]
  O> set bad first filter [ x -> [who] of item 0 x = 1 ] r
  O> set good filter [ x -> [who] of item 0 x != 1 ] r
  item 2 bad => false
  is-string? item 1 bad => true
  member? "simulated provider failure" item 1 bad => true
  map [ x -> item 2 x ] good => [true true]
  [llm:history] of turtle 1 => []
  [length llm:history] of turtle 0 => 2
  O> clear-all

LLMChatAllRecordsUsage
  extensions [llm]
  globals [r]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> clear-all
  O> crt 2
  O> set r llm:chat-all turtles [ -> "__TEST_USAGE:4,3 hi" ]
  [llm:get llm:usage "total-tokens"] of turtle 0 => 7
  [llm:get llm:usage "calls"] of turtle 1 => 1
  llm:get llm:usage-total "calls" => 2
  llm:get llm:usage-total "total-tokens" => 14
  O> clear-all

LLMChatAllUsesAgentProfile
  extensions [llm]
  globals [r]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> llm:set-model "model-default"
  O> clear-all
  O> llm:load-profile "a" "demos/test-profile-a"
  O> crt 2
  O> ask turtle 0 [ llm:use-profile "a" ]
  O> set r llm:chat-all turtles [ -> "__TEST_ECHO_MODEL" ]
  item 1 first filter [ x -> [who] of item 0 x = 0 ] r => "model-a"
  item 1 first filter [ x -> [who] of item 0 x = 1 ] r => "model-default"
  O> clear-all

LLMChatAllWithSchema
  extensions [llm]
  globals [r schema]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> clear-all
  O> set schema "{\"type\":\"object\",\"properties\":{\"n\":{\"type\":\"number\"}}}"
  O> crt 2
  O> set r (llm:chat-all turtles [ -> ifelse-value who = 0 [ "__TEST_RESPOND:{\"n\": 5}" ] [ "__TEST_RESPOND:not json" ] ] schema)
  llm:get item 1 first filter [ x -> [who] of item 0 x = 0 ] r "n" => 5
  item 2 first filter [ x -> [who] of item 0 x = 0 ] r => true
  item 2 first filter [ x -> [who] of item 0 x = 1 ] r => false
  [llm:history] of turtle 1 => []
  O> clear-all

LLMChatAllBadSchemaFailsFast
  extensions [llm]
  globals [r err]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> clear-all
  O> crt 2
  O> carefully [ set r (llm:chat-all turtles [ -> "hi" ] "{\"type\":\"string\"}") ] [ set err error-message ]
  member? "type 'object'" err => true
  llm:get llm:usage-total "calls" => 0
  O> clear-all

LLMChatAllNonStringPromptErrors
  extensions [llm]
  globals [r err]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> clear-all
  O> crt 2
  O> carefully [ set r llm:chat-all turtles [ -> who ] ] [ set err error-message ]
  member? "must report a string" err => true
  llm:get llm:usage-total "calls" => 0
  O> clear-all

LLMChatAllEmptyAgentset
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> clear-all
  llm:chat-all turtles [ -> "hi" ] => []

LLMChatAllRunsConcurrently
  extensions [llm]
  globals [r elapsed]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> clear-all
  O> crt 10
  O> reset-timer
  O> set r llm:chat-all turtles [ -> "__TEST_DELAY:400:hi" ]
  O> set elapsed timer
  map [ x -> item 2 x ] r => [true true true true true true true true true true]
  elapsed < 2 => true
  O> clear-all

LLMChatAllHungBatchWaitsOneBudget
  extensions [llm]
  globals [r elapsed]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> clear-all
  O> llm:load-config "demos/test-config-fast-timeout"
  O> crt 3
  O> reset-timer
  O> set r llm:chat-all turtles [ -> "__TEST_DELAY:3000:hi" ]
  O> set elapsed timer
  map [ x -> item 2 x ] r => [false false false]
  elapsed < 1.8 => true
  O> wait 2.5
  [llm:history] of turtle 0 => []
  [llm:history] of turtle 2 => []
  O> clear-all

LLMChatAllProgressExtendsTheWait
  extensions [llm]
  globals [r]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> clear-all
  O> llm:load-config "demos/test-config-fast-timeout"
  O> crt 2
  O> set r llm:chat-all turtles [ -> ifelse-value who = 0 [ "__TEST_DELAY:600:a" ] [ "__TEST_DELAY:1400:b" ] ]
  map [ x -> item 2 x ] r => [true true]
  O> clear-all

LLMChatAllRejectsExtraSchemas
  extensions [llm]
  globals [r err]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> clear-all
  O> crt 1
  O> carefully [ set r (llm:chat-all turtles [ -> "hi" ] "{\"type\":\"object\"}" "{\"type\":\"object\"}") ] [ set err error-message ]
  member? "at most one schema" err => true
  O> clear-all

LLMRefusalRaisesAfterRecordingUsage
  extensions [llm]
  globals [r err]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> clear-all
  O> llm:set-history [["user" "prior"] ["assistant" "context"]]
  O> carefully [ set r llm:chat "__TEST_USAGE:5,2 __TEST_REFUSE:cannot help with that" ] [ set err error-message ]
  err => "Extension exception: model refused: cannot help with that"
  llm:get llm:usage-total "calls" => 1
  llm:get llm:usage "input-tokens" => 5
  llm:history => [["user" "prior"] ["assistant" "context"]]
  O> clear-all

LLMRefusalRaisesFromEveryChatPrimitive
  extensions [llm]
  globals [p r e2 e3 e4 e5 e6]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> clear-all
  O> set p llm:chat-async "__TEST_REFUSE:no async"
  O> wait 0.3
  llm:get llm:usage-total "calls" => 1
  llm:history => []
  O> carefully [ set r llm:chat-with-template "demos/simple-template.yaml" [["text" "__TEST_REFUSE:no template"]] ] [ set e2 error-message ]
  member? "model refused: no template" e2 => true
  O> carefully [ set r llm:choose "__TEST_REFUSE:no choice" ["north" "south"] ] [ set e3 error-message ]
  member? "model refused: no choice" e3 => true
  O> carefully [ set r llm:chat-with-schema "__TEST_REFUSE:no schema" "{\"type\":\"object\",\"properties\":{\"a\":{\"type\":\"string\"}}}" ] [ set e4 error-message ]
  member? "model refused: no schema" e4 => true
  O> carefully [ set r llm:chat-json "__TEST_REFUSE:no json" ] [ set e5 error-message ]
  member? "model refused: no json" e5 => true
  O> carefully [ set r llm:chat-with-thinking "__TEST_REFUSE:no thinking" ] [ set e6 error-message ]
  member? "model refused: no thinking" e6 => true
  llm:get llm:usage-total "calls" => 6
  llm:history => []
  O> crt 1
  O> set r llm:chat-all turtles [ -> "__TEST_REFUSE:no batch" ]
  item 1 (item 0 r) => "model refused: no batch"
  item 2 (item 0 r) => false
  [llm:history] of turtle 0 => []
  llm:get llm:usage-total "calls" => 7

LLMChatAsyncAfterClearAllWritesNothing
  extensions [llm]
  globals [r]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> clear-all
  O> let p llm:chat-async "__TEST_USAGE:4,3 __TEST_DELAY:300:slow" clear-all set r (runresult p)
  r => "stub:slow"
  llm:history => []
  llm:get llm:usage "calls" => 0
  llm:get llm:usage-total "calls" => 0
  O> clear-all

LLMChatAsyncAfterClearAllSkipsUnawaitedReply
  extensions [llm]
  globals [p]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> clear-all
  O> let q llm:chat-async "__TEST_USAGE:4,3 __TEST_DELAY:300:slow" clear-all set p q
  O> wait 0.6
  llm:history => []
  llm:get llm:usage-total "calls" => 0
  O> clear-all

LLMDecideChoice
  extensions [llm]
  globals [r]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "typesafe"
  O> clear-all
  item 0 llm:active => "typesafe"
  item 1 llm:active => "jev-latest"
  O> set r llm:decide "__TEST_DECIDE:stay:0.5 bar tonight?" ["choice" ["move" "stay" "wait"]]
  r => ["stay" 0.5 [["move" 0.25] ["stay" 0.5] ["wait" 0.25]]]
  llm:decide "bar tonight?" ["choice" ["move" "stay"]] => ["move" 1 [["move" 1] ["stay" 0]]]
  llm:history => []
  O> clear-all

LLMDecideScore
  extensions [llm]
  globals [r]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "typesafe"
  O> clear-all
  O> set r llm:decide "__TEST_DECIDE:2:0.5 how crowded?" ["score" ["low" "medium" "high"]]
  r => [1.25 0.5 [["low" 0.25] ["medium" 0.25] ["high" 0.5]]]
  is-number? item 0 r => true
  llm:history => []
  O> clear-all

LLMDecideYesNo
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "typesafe"
  O> clear-all
  llm:decide "__TEST_DECIDE:true:0.75 go?" ["yes-no"] => [true 0.75 [[true 0.75] [false 0.25]]]
  llm:decide "__TEST_DECIDE:false:0.75 go?" ["yes-no"] => [false 0.75 [[true 0.25] [false 0.75]]]
  llm:decide "__TEST_DECIDE:true:0.56 go?" ["yes-no"] => [true 0.56 [[true 0.56] [false 0.44]]]
  llm:history => []
  O> clear-all

LLMDecideRecordsUsageUntilClearAll
  extensions [llm]
  globals [r]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "typesafe"
  O> clear-all
  O> set r llm:decide "__TEST_USAGE:5,1 go?" ["yes-no"]
  O> set r llm:decide "__TEST_USAGE:2,0 go?" ["choice" ["a" "b"]]
  llm:get llm:usage "input-tokens" => 7
  llm:get llm:usage "calls" => 2
  llm:get llm:usage-total "calls" => 2
  O> clear-all
  llm:get llm:usage "calls" => 0
  llm:get llm:usage-total "calls" => 0

LLMDecideSpecValidation
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "typesafe"
  O> clear-all
  llm:decide "x" [] => ERROR Extension exception: llm:decide: spec must be ["choice" [options]], ["score" [levels]] or ["yes-no"]
  llm:decide "x" ["vote" ["a"]] => ERROR Extension exception: llm:decide: unknown decision type "vote"; use "choice", "score" or "yes-no"
  llm:decide "x" ["choice" []] => ERROR Extension exception: llm:decide: "choice" needs a list of 1 to 255 options
  llm:decide "x" ["choice" "a"] => ERROR Extension exception: llm:decide: "choice" needs a list of 1 to 255 options
  llm:decide "x" ["choice" ["a" "a"]] => ERROR Extension exception: llm:decide: "choice" options must be distinct, but "a" appears more than once
  llm:decide "x" ["choice" ["a" ""]] => ERROR Extension exception: llm:decide: "choice" options must be non-empty strings
  llm:decide "x" ["choice" ["a" 3]] => ERROR Extension exception: llm:decide: "choice" options must be non-empty strings
  llm:decide "x" ["score" ["only"]] => ERROR Extension exception: llm:decide: "score" needs a list of 2 to 10 levels
  llm:decide "x" ["score" ["a" "b" "c" "d" "e" "f" "g" "h" "i" "j" "k"]] => ERROR Extension exception: llm:decide: "score" needs a list of 2 to 10 levels
  llm:decide "x" ["score" ["a" ""]] => ERROR Extension exception: llm:decide: "score" levels must be non-empty strings
  llm:decide "x" ["yes-no" ["a"]] => ERROR Extension exception: llm:decide: "yes-no" takes no list
  O> crt 1
  llm:decide-all turtles [ -> "x" ] ["vote"] => ERROR Extension exception: llm:decide-all: unknown decision type "vote"; use "choice", "score" or "yes-no"
  llm:get llm:usage-total "calls" => 0

LLMDecideNeedsDecisionProvider
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> clear-all
  llm:decide "x" ["yes-no"] => ERROR Extension exception: provider "openai" has no decision API; use llm:choose, or llm:set-provider "typesafe"
  O> crt 2
  llm:decide-all turtles [ -> "x" ] ["yes-no"] => ERROR Extension exception: provider "openai" has no decision API; use llm:choose, or llm:set-provider "typesafe"
  llm:get llm:usage-total "calls" => 0

LLMDecideAllMixedResults
  extensions [llm]
  globals [r bad good]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "typesafe"
  O> clear-all
  O> crt 3
  O> set r llm:decide-all turtles [ -> ifelse-value who = 1 [ "__TEST_FAIL" ] [ "__TEST_USAGE:2,0 __TEST_DECIDE:stay:0.75 go?" ] ] ["choice" ["move" "stay"]]
  length r => 3
  map [ x -> length x ] r => [3 3 3]
  sort map [ x -> [who] of item 0 x ] r => [0 1 2]
  O> set bad first filter [ x -> [who] of item 0 x = 1 ] r
  O> set good filter [ x -> [who] of item 0 x != 1 ] r
  item 2 bad => false
  member? "simulated provider failure" item 1 bad => true
  map [ x -> item 2 x ] good => [true true]
  map [ x -> item 1 x ] good => [["stay" 0.75 [["move" 0.25] ["stay" 0.75]]] ["stay" 0.75 [["move" 0.25] ["stay" 0.75]]]]
  [llm:get llm:usage "input-tokens"] of turtle 0 => 2
  [llm:get llm:usage "calls"] of turtle 1 => 0
  llm:get llm:usage-total "calls" => 2
  [llm:history] of turtle 0 => []
  O> clear-all

LLMDecideAllPromptRunsAsEachAgent
  extensions [llm]
  globals [r]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "typesafe"
  O> clear-all
  O> crt 2
  O> set r llm:decide-all turtles [ -> (word "__TEST_DECIDE:" item who ["true" "false"] ":1 go?") ] ["yes-no"]
  reduce and map [ x -> item 0 item 1 x = ([who] of item 0 x = 0) ] r => true
  llm:decide-all no-turtles [ -> "x" ] ["yes-no"] => []
  O> clear-all

LLMChooseOnDecisionProvider
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "typesafe"
  O> clear-all
  llm:choose "__TEST_USAGE:3,0 __TEST_DECIDE:south:0.9 where to?" ["north" "south"] => "south"
  llm:choose "where to?" ["north" "south"] => "north"
  llm:history => []
  llm:get llm:usage "input-tokens" => 3
  llm:get llm:usage "calls" => 2
  O> clear-all

LLMTextPrimitivesOnDecisionProvider
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "typesafe"
  O> clear-all
  llm:chat "hi" => ERROR Extension exception: typesafe (Jev) returns decisions, not text; use llm:decide, llm:decide-all or llm:choose
  llm:chat-async "hi" => ERROR Extension exception: typesafe (Jev) returns decisions, not text; use llm:decide, llm:decide-all or llm:choose
  llm:chat-with-thinking "hi" => ERROR Extension exception: typesafe (Jev) returns decisions, not text; use llm:decide, llm:decide-all or llm:choose
  llm:chat-json "hi" => ERROR Extension exception: typesafe (Jev) returns decisions, not text; use llm:decide, llm:decide-all or llm:choose
  llm:chat-with-schema "hi" "{\"type\":\"object\",\"properties\":{\"a\":{\"type\":\"string\"}}}" => ERROR Extension exception: typesafe (Jev) returns decisions, not text; use llm:decide, llm:decide-all or llm:choose
  llm:chat-with-template "demos/simple-template.yaml" [["text" "hi"]] => ERROR Extension exception: typesafe (Jev) returns decisions, not text; use llm:decide, llm:decide-all or llm:choose
  O> crt 1
  llm:chat-all turtles [ -> "hi" ] => ERROR Extension exception: typesafe (Jev) returns decisions, not text; use llm:decide, llm:decide-all or llm:choose
  llm:history => []
  llm:get llm:usage-total "calls" => 0

LLMTypeSafeProviderInfo
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "typesafe"
  empty? (llm:provider-help "typesafe") => false
  member? "typesafe" llm:providers => true
  (position "jev-preview" llm:list-models) != false => true
  item 1 first filter [ x -> first x = "typesafe" ] llm:provider-status => ["ready" true]
  item 2 first filter [ x -> first x = "typesafe" ] llm:provider-status => ["has-key" true]
  O> llm:set-model "jev-1.14.0"
  item 1 llm:active => "jev-1.14.0"

LLMChooseOnDecisionProviderValidatesOptions
  extensions [llm]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "typesafe"
  O> clear-all
  llm:choose "x" ["a" "a"] => ERROR Extension exception: llm:choose: "choice" options must be distinct, but "a" appears more than once
  llm:choose "x" ["a" ""] => ERROR Extension exception: llm:choose: "choice" options must be non-empty strings
  llm:choose "x" n-values 256 [ i -> (word "o" i) ] => ERROR Extension exception: llm:choose: "choice" needs a list of 1 to 255 options
  llm:get llm:usage-total "calls" => 0
  llm:choose "__TEST_DECIDE:west:0.9 where to?" ["north" "south"] => ERROR Extension exception: llm:choose: provider picked "west", which is not one of the options: north, south
  llm:history => []
  O> clear-all

LLMEmptyReplyAtTokenLimitRaisesAfterRecordingUsage
  extensions [llm]
  globals [r err]
  O> llm:set-api-key "test-key"
  O> llm:set-provider "openai"
  O> clear-all
  O> llm:set-history [["user" "prior"] ["assistant" "context"]]
  O> carefully [ set r llm:chat "__TEST_USAGE:5,4000,4000 __TEST_TRUNCATE" ] [ set err error-message ]
  member? "empty reply" err => true
  member? "max_tokens" err => true
  member? "4000 output tokens" err => true
  llm:get llm:usage-total "calls" => 1
  llm:get llm:usage-total "output-tokens" => 4000
  llm:history => [["user" "prior"] ["assistant" "context"]]
  O> clear-all