Skip to content

Commit 2423af8

Browse files
committed
feat(llmrails): enable stream_usage only for supported engines
1 parent 099586b commit 2423af8

File tree

1 file changed

+8
-1
lines changed

1 file changed

+8
-1
lines changed

nemoguardrails/rails/llm/llmrails.py

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -367,7 +367,14 @@ def _prepare_model_kwargs(self, model_config):
367367
if api_key:
368368
kwargs["api_key"] = api_key
369369

370-
kwargs["stream_usage"] = True
370+
# enable streaming token usage when streaming is enabled
371+
# this is primarily supported by OpenAI and NIM providers
372+
if self.config.streaming and model_config.engine in [
373+
"openai",
374+
"azure_openai",
375+
"nim",
376+
]:
377+
kwargs["stream_usage"] = True
371378

372379
return kwargs
373380

0 commit comments

Comments
 (0)