Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 21 additions & 1 deletion sdk/go/ai/tool_calling.go
Original file line number Diff line number Diff line change
Expand Up @@ -271,8 +271,28 @@ func (c *Client) ExecuteToolCallLoopResult(
trace.TotalToolCalls = totalCalls

var args map[string]interface{}
if err := json.Unmarshal([]byte(tc.Function.Arguments), &args); err != nil {
if strings.TrimSpace(tc.Function.Arguments) == "" {
// Some providers send "" for tools that take no arguments.
args = map[string]interface{}{}
} else if err := json.Unmarshal([]byte(tc.Function.Arguments), &args); err != nil || args == nil {
// Malformed or truncated arguments: do not run the tool with
// empty args; report the error so the model can retry.
if err == nil {
err = fmt.Errorf("arguments must be a JSON object")
}
invalidMsg := Message{
Role: "tool",
Content: []ContentPart{{Type: "text", Text: encodeToolContent(map[string]string{
"error": fmt.Sprintf("Tool call to '%s' has invalid or truncated JSON arguments (%v). Please retry with valid JSON arguments.", tc.Function.Name, err),
})}},
ToolCallID: tc.ID,
}
loopMessages = append(loopMessages, invalidMsg)
trace.Messages = append(trace.Messages, TracedMessage{
Message: invalidMsg,
Source: TraceSourceToolError,
})
continue
}
toolName := unsanitizeToolName(tc.Function.Name)

Expand Down
64 changes: 60 additions & 4 deletions sdk/go/ai/tool_calling_loop_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ func TestExecuteToolCallLoop_CompletesAfterToolResponse(t *testing.T) {
assert.Equal(t, map[string]interface{}{"status": "open"}, trace.Calls[0].Result)
}

func TestExecuteToolCallLoop_RecordsToolErrorsAndMalformedArguments(t *testing.T) {
func TestExecuteToolCallLoop_RecordsToolErrors(t *testing.T) {
var requestCount atomic.Int32
client := newToolLoopClient(t, func(w http.ResponseWriter, r *http.Request) {
count := requestCount.Add(1)
Expand All @@ -106,9 +106,9 @@ func TestExecuteToolCallLoop_RecordsToolErrorsAndMalformedArguments(t *testing.T
Message: Message{
Role: "assistant",
ToolCalls: []ToolCall{{
ID: "call-2",
Type: "function",
Function: ToolCallFunction{Name: "lookup", Arguments: `{bad json`},
ID: "call-2",
Type: "function",
Function: ToolCallFunction{Name: "lookup", Arguments: ``},
}},
},
FinishReason: "tool_calls",
Expand Down Expand Up @@ -152,6 +152,62 @@ func TestExecuteToolCallLoop_RecordsToolErrorsAndMalformedArguments(t *testing.T
assert.Empty(t, trace.Calls[0].Arguments)
}

func TestExecuteToolCallLoop_MalformedArgumentsAreNotExecuted(t *testing.T) {
var requestCount atomic.Int32
client := newToolLoopClient(t, func(w http.ResponseWriter, r *http.Request) {
count := requestCount.Add(1)
if count == 1 {
require.NoError(t, json.NewEncoder(w).Encode(Response{
Choices: []Choice{{
Message: Message{
Role: "assistant",
ToolCalls: []ToolCall{{
ID: "call-3",
Type: "function",
Function: ToolCallFunction{Name: "lookup", Arguments: `{"query": "trunca`},
}},
},
FinishReason: "tool_calls",
}},
}))
return
}

var req Request
require.NoError(t, json.NewDecoder(r.Body).Decode(&req))
require.Len(t, req.Messages, 3)
assert.Equal(t, "tool", req.Messages[2].Role)
assert.Equal(t, "call-3", req.Messages[2].ToolCallID)
assert.Contains(t, req.Messages[2].Content[0].Text, "invalid or truncated JSON arguments")
require.NoError(t, json.NewEncoder(w).Encode(Response{
Choices: []Choice{{
Message: Message{
Role: "assistant",
Content: []ContentPart{{Type: "text", Text: "retry later"}},
},
FinishReason: "stop",
}},
}))
})

called := false
resp, trace, err := client.ExecuteToolCallLoop(
context.Background(),
[]Message{{Role: "user", Content: []ContentPart{{Type: "text", Text: "Lookup"}}}},
[]ToolDefinition{{Type: "function", Function: ToolFunction{Name: "lookup", Parameters: map[string]interface{}{"type": "object"}}}},
ToolCallConfig{MaxTurns: 3, MaxToolCalls: 2},
func(_ context.Context, _ string, _ map[string]interface{}) (map[string]interface{}, error) {
called = true
return map[string]interface{}{}, nil
},
)

require.NoError(t, err)
assert.Equal(t, "retry later", resp.Text())
assert.False(t, called, "tool must not run with malformed arguments")
assert.Empty(t, trace.Calls)
}

func TestSimpleAndStructuredAIAndResponseHelpers(t *testing.T) {
var requestCount atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
Expand Down
27 changes: 24 additions & 3 deletions sdk/python/agentfield/tool_calling.py
Original file line number Diff line number Diff line change
Expand Up @@ -546,9 +546,30 @@ async def execute_tool_call_loop(
)
continue
try:
func_args = json.loads(raw_args)
except (json.JSONDecodeError, TypeError):
func_args = {}
# Some providers send "" for tools that take no arguments.
func_args = json.loads(raw_args) if raw_args.strip() else {}
if not isinstance(func_args, dict):
raise TypeError("tool arguments must be a JSON object")
except (json.JSONDecodeError, TypeError, AttributeError) as e:
# Malformed or truncated arguments: do not run the tool with
# empty args; report the error so the model can retry.
log_warn(f"Invalid JSON arguments for tool call '{func_name}': {e}")
invalid_args_message = {
"role": "tool",
"tool_call_id": tc.id,
"content": json.dumps(
{
"error": f"Tool call to '{func_name}' has invalid or truncated JSON arguments ({e}). Please retry with valid JSON arguments."
}
),
}
messages.append(invalid_args_message)
trace.messages.append(
TracedMessage(
message=invalid_args_message, source=TRACE_SOURCE_TOOL_ERROR
)
)
continue

record = ToolCallRecord(
tool_name=func_name,
Expand Down
69 changes: 69 additions & 0 deletions sdk/python/tests/test_tool_calling.py
Original file line number Diff line number Diff line change
Expand Up @@ -456,6 +456,75 @@ async def mock_completion(params):
== "Tool call to 'sentiment_agent.analyze' is missing the 'arguments' field. Please retry with valid JSON arguments."
)

@pytest.mark.asyncio
@pytest.mark.parametrize("bad_args", ['{"text": "trunca', "not json", "[1, 2]"])
async def test_malformed_arguments_are_not_executed(self, bad_args):
agent = make_mock_agent()
agent.call = AsyncMock(return_value={"result": "ok"})

messages = [{"role": "user", "content": "test"}]
tools = capabilities_to_tool_schemas([make_reasoner()])
config = ToolCallConfig(max_turns=5)

tool_resp = make_llm_response(tool_calls=[make_tool_call(arguments=bad_args)])
final_resp = make_llm_response(content="Please retry")
call_count = 0

async def mock_completion(params):
nonlocal call_count
call_count += 1
return tool_resp if call_count == 1 else final_resp

resp, trace = await execute_tool_call_loop(
agent=agent,
messages=messages,
tools=tools,
config=config,
needs_lazy_hydration=False,
litellm_params={"model": "openai/gpt-4"},
make_completion=mock_completion,
)

agent.call.assert_not_called()
assert trace.calls == []
tool_messages = [m for m in messages if m.get("role") == "tool"]
assert len(tool_messages) == 1
assert (
"invalid or truncated JSON arguments"
in json.loads(tool_messages[0]["content"])["error"]
)

@pytest.mark.asyncio
async def test_empty_string_arguments_run_with_no_args(self):
agent = make_mock_agent()
agent.call = AsyncMock(return_value={"result": "ok"})

messages = [{"role": "user", "content": "test"}]
tools = capabilities_to_tool_schemas([make_reasoner()])
config = ToolCallConfig(max_turns=5)

tool_resp = make_llm_response(tool_calls=[make_tool_call(arguments="")])
final_resp = make_llm_response(content="Done")
call_count = 0

async def mock_completion(params):
nonlocal call_count
call_count += 1
return tool_resp if call_count == 1 else final_resp

await execute_tool_call_loop(
agent=agent,
messages=messages,
tools=tools,
config=config,
needs_lazy_hydration=False,
litellm_params={"model": "openai/gpt-4"},
make_completion=mock_completion,
)

agent.call.assert_awaited_once()
assert agent.call.await_args.kwargs == {}

@pytest.mark.asyncio
async def test_max_tool_calls_limit(self):
agent = make_mock_agent()
Expand Down
10 changes: 6 additions & 4 deletions sdk/python/tests/test_tool_calling_error_paths.py
Original file line number Diff line number Diff line change
Expand Up @@ -129,12 +129,13 @@ async def test_invalid_argument_type_is_reported_and_loop_continues():
)

assert trace.total_tool_calls == 1
assert trace.calls[0].error is not None
assert "mapping" in trace.calls[0].error
# Invalid arguments are reported to the model without creating a call record.
assert trace.calls == []
assert agent.call.await_count == 0
tool_messages = [m for m in messages if m.get("role") == "tool"]
assert len(tool_messages) == 1
assert "error" in json.loads(tool_messages[0]["content"])
error = json.loads(tool_messages[0]["content"])["error"]
assert "JSON object" in error


@pytest.mark.asyncio
Expand Down Expand Up @@ -164,11 +165,12 @@ async def test_mixed_valid_and_invalid_tool_calls_in_single_turn():
)

assert trace.total_tool_calls == 2
assert len(trace.calls) == 1
assert trace.calls[0].result == {"echoed": "hi"}
assert trace.calls[1].error is not None
agent.call.assert_awaited_once_with("utility.echo", text="hi")
tool_messages = [m for m in messages if m.get("role") == "tool"]
assert len(tool_messages) == 2
assert "invalid" in json.loads(tool_messages[1]["content"])["error"]


@pytest.mark.asyncio
Expand Down
Loading