From 8ffbdc80e49419f63f70af0bd227fc6527f3aba6 Mon Sep 17 00:00:00 2001 From: Tarun Kumar Date: Wed, 16 Apr 2025 18:00:46 +0530 Subject: [PATCH 1/7] Add tests for openai schema testing Signed-off-by: Tarun Kumar --- requirements/test.in | 1 + requirements/test.txt | 88 +++++++++++++++++-- .../entrypoints/openai/test_openai_schema.py | 44 ++++++++++ 3 files changed, 128 insertions(+), 5 deletions(-) create mode 100644 tests/entrypoints/openai/test_openai_schema.py diff --git a/requirements/test.in b/requirements/test.in index b9b3df0651b..b259157dac9 100644 --- a/requirements/test.in +++ b/requirements/test.in @@ -35,6 +35,7 @@ datamodel_code_generator # required for minicpm3 test lm-eval[api]==0.4.8 # required for model evaluation test transformers==4.51.1 huggingface-hub[hf_xet]>=0.30.0 # Required for Xet downloads. +schemathesis>=3.39.15 # Required for openai schema test. # quantization bitsandbytes>=0.45.3 buildkite-test-collector==0.1.9 diff --git a/requirements/test.txt b/requirements/test.txt index a5c062b0b1f..a04c481d1ed 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -20,21 +20,29 @@ aiosignal==1.3.1 annotated-types==0.7.0 # via pydantic anyio==4.6.2.post1 - # via httpx + # via + # httpx + # starlette argcomplete==3.5.1 # via datamodel-code-generator +arrow==1.3.0 + # via isoduration attrs==24.2.0 # via # aiohttp + # hypothesis # jsonlines # jsonschema + # pytest-subtests # referencing audioread==3.0.1 # via librosa awscli==1.35.23 # via -r requirements/test.in backoff==2.2.1 - # via -r requirements/test.in + # via + # -r requirements/test.in + # schemathesis bitsandbytes==0.45.3 # via -r requirements/test.in black==24.10.0 @@ -69,11 +77,13 @@ click==8.1.7 # jiwer # nltk # ray + # schemathesis # typer colorama==0.4.6 # via # awscli # sacrebleu + # schemathesis # tqdm-multiprocess contourpy==1.3.0 # via matplotlib @@ -137,6 +147,8 @@ filelock==3.16.1 # transformers fonttools==4.54.1 # via matplotlib +fqdn==1.5.1 + # via jsonschema frozendict==2.4.6 # via einx frozenlist==1.5.0 @@ -155,8 +167,12 @@ genai-perf==0.0.8 # via -r requirements/test.in genson==1.3.0 # via datamodel-code-generator +graphql-core==3.2.6 + # via hypothesis-graphql h11==0.14.0 # via httpcore +harfile==0.3.0 + # via schemathesis hf-xet==0.1.4 # via huggingface-hub hiredis==3.0.0 @@ -164,7 +180,9 @@ hiredis==3.0.0 httpcore==1.0.6 # via httpx httpx==0.27.2 - # via -r requirements/test.in + # via + # -r requirements/test.in + # schemathesis huggingface-hub==0.30.1 # via # -r requirements/test.in @@ -179,17 +197,29 @@ huggingface-hub==0.30.1 # vocos humanize==4.11.0 # via runai-model-streamer +hypothesis==6.131.0 + # via + # hypothesis-graphql + # hypothesis-jsonschema + # schemathesis +hypothesis-graphql==0.11.1 + # via schemathesis +hypothesis-jsonschema==0.23.1 + # via schemathesis idna==3.10 # via # anyio # email-validator # httpx + # jsonschema # requests # yarl inflect==5.6.2 # via datamodel-code-generator iniconfig==2.0.0 # via pytest +isoduration==20.11.0 + # via jsonschema isort==5.13.2 # via datamodel-code-generator jinja2==3.1.6 @@ -209,12 +239,18 @@ joblib==1.4.2 # scikit-learn jsonlines==4.0.0 # via lm-eval +jsonpointer==3.0.0 + # via jsonschema jsonschema==4.23.0 # via + # hypothesis-jsonschema # mistral-common # ray + # schemathesis jsonschema-specifications==2024.10.1 # via jsonschema +junit-xml==1.9 + # via schemathesis kaleido==0.2.1 # via genai-perf kiwisolver==1.4.7 @@ -236,7 +272,9 @@ lxml==5.3.0 markdown-it-py==3.0.0 # via rich markupsafe==3.0.2 - # via jinja2 + # via + # jinja2 + # werkzeug matplotlib==3.9.2 # via -r requirements/test.in mbstrdecoder==1.1.3 @@ -443,6 +481,8 @@ pygments==2.18.0 # via rich pyparsing==3.2.0 # via matplotlib +pyrate-limiter==3.7.0 + # via schemathesis pytablewriter==1.2.0 # via lm-eval pytest==8.3.3 @@ -455,7 +495,9 @@ pytest==8.3.3 # pytest-mock # pytest-rerunfailures # pytest-shard + # pytest-subtests # pytest-timeout + # schemathesis pytest-asyncio==0.24.0 # via -r requirements/test.in pytest-forked==1.6.0 @@ -466,10 +508,13 @@ pytest-rerunfailures==14.0 # via -r requirements/test.in pytest-shard==0.1.2 # via -r requirements/test.in +pytest-subtests==0.14.1 + # via schemathesis pytest-timeout==2.3.1 # via -r requirements/test.in python-dateutil==2.9.0.post0 # via + # arrow # botocore # matplotlib # pandas @@ -491,6 +536,7 @@ pyyaml==6.0.2 # peft # ray # responses + # schemathesis # timm # transformers # vocos @@ -521,10 +567,16 @@ requests==2.32.3 # pooch # ray # responses + # schemathesis + # starlette-testclient # tiktoken # transformers responses==0.25.3 # via genai-perf +rfc3339-validator==0.1.4 + # via jsonschema +rfc3987==1.3.8 + # via jsonschema rich==13.9.4 # via # genai-perf @@ -553,6 +605,8 @@ safetensors==0.4.5 # peft # timm # transformers +schemathesis==3.39.15 + # via -r requirements/test.in scikit-learn==1.5.2 # via # librosa @@ -577,12 +631,16 @@ shellingham==1.5.4 # via typer six==1.16.0 # via + # junit-xml # python-dateutil + # rfc3339-validator # rouge-score sniffio==1.3.1 # via # anyio # httpx +sortedcontainers==2.4.0 + # via hypothesis soundfile==0.12.1 # via # -r requirements/test.in @@ -591,6 +649,12 @@ soxr==0.5.0.post1 # via librosa sqlitedict==2.1.0 # via lm-eval +starlette==0.46.2 + # via + # schemathesis + # starlette-testclient +starlette-testclient==0.4.1 + # via schemathesis statsmodels==0.14.4 # via genai-perf sympy==1.13.1 @@ -619,6 +683,10 @@ timm==1.0.11 # via -r requirements/test.in tokenizers==0.21.0 # via transformers +tomli==2.2.1 + # via schemathesis +tomli-w==1.2.0 + # via schemathesis torch==2.6.0 # via # -r requirements/test.in @@ -682,6 +750,8 @@ typepy==1.3.2 # tabledata typer==0.15.2 # via fastsafetensors +types-python-dateutil==2.9.0.20241206 + # via arrow typing-extensions==4.12.2 # via # huggingface-hub @@ -694,6 +764,8 @@ typing-extensions==4.12.2 # typer tzdata==2024.2 # via pandas +uri-template==1.3.0 + # via jsonschema urllib3==2.2.3 # via # blobfile @@ -705,6 +777,10 @@ vector-quantize-pytorch==1.21.2 # via -r requirements/test.in vocos==0.1.0 # via -r requirements/test.in +webcolors==24.11.1 + # via jsonschema +werkzeug==3.1.3 + # via schemathesis word2number==1.1 # via lm-eval xxhash==3.5.0 @@ -712,6 +788,8 @@ xxhash==3.5.0 # datasets # evaluate yarl==1.17.1 - # via aiohttp + # via + # aiohttp + # schemathesis zstandard==0.23.0 # via lm-eval diff --git a/tests/entrypoints/openai/test_openai_schema.py b/tests/entrypoints/openai/test_openai_schema.py new file mode 100644 index 00000000000..0a776ae9bbc --- /dev/null +++ b/tests/entrypoints/openai/test_openai_schema.py @@ -0,0 +1,44 @@ +import pytest +import schemathesis +import pytest +from schemathesis import GenerationConfig +from ...utils import RemoteOpenAIServer +schemathesis.experimental.OPEN_API_3_1.enable() + +MODEL_NAME = "ibm-granite/granite-vision-3.2-2b" +MAXIMUM_IMAGES = 2 + +@pytest.fixture(scope="module") +def server(): + args = [ + "--task", + "generate", + "--max-model-len", + "2048", + "--max-num-seqs", + "5", + "--enforce-eager", + "--trust-remote-code", + "--limit-mm-per-prompt", + f"image={MAXIMUM_IMAGES}", + ] + + with RemoteOpenAIServer(MODEL_NAME, args) as remote_server: + yield remote_server + +@pytest.fixture(scope="module") +def get_schema(server): + # avoid generating null (\x00) bytes in strings during test case generation + return schemathesis.openapi.from_uri(f"{server.url_root}/openapi.json",generation_config=GenerationConfig(allow_x00=False),) + +schema = schemathesis.from_pytest_fixture("get_schema") + + +@schema.parametrize() +async def test_openapi_stateless(case): + case.headers = { + "Content-Type": "application/json", + } + #disable SSL certificate verification for localhost it doesn't matter + await case.call_and_validate(verify=False) + \ No newline at end of file From a1ae7db9a1d99ed06271169edd8f36b7323eca67 Mon Sep 17 00:00:00 2001 From: Tarun Kumar Date: Wed, 16 Apr 2025 18:28:48 +0530 Subject: [PATCH 2/7] fix precommit Signed-off-by: Tarun Kumar --- tests/entrypoints/openai/test_openai_schema.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/entrypoints/openai/test_openai_schema.py b/tests/entrypoints/openai/test_openai_schema.py index 0a776ae9bbc..d241dd0baf7 100644 --- a/tests/entrypoints/openai/test_openai_schema.py +++ b/tests/entrypoints/openai/test_openai_schema.py @@ -1,13 +1,16 @@ +# SPDX-License-Identifier: Apache-2.0 import pytest import schemathesis -import pytest from schemathesis import GenerationConfig + from ...utils import RemoteOpenAIServer + schemathesis.experimental.OPEN_API_3_1.enable() MODEL_NAME = "ibm-granite/granite-vision-3.2-2b" MAXIMUM_IMAGES = 2 + @pytest.fixture(scope="module") def server(): args = [ @@ -26,10 +29,15 @@ def server(): with RemoteOpenAIServer(MODEL_NAME, args) as remote_server: yield remote_server + @pytest.fixture(scope="module") def get_schema(server): # avoid generating null (\x00) bytes in strings during test case generation - return schemathesis.openapi.from_uri(f"{server.url_root}/openapi.json",generation_config=GenerationConfig(allow_x00=False),) + return schemathesis.openapi.from_uri( + f"{server.url_root}/openapi.json", + generation_config=GenerationConfig(allow_x00=False), + ) + schema = schemathesis.from_pytest_fixture("get_schema") @@ -39,6 +47,5 @@ async def test_openapi_stateless(case): case.headers = { "Content-Type": "application/json", } - #disable SSL certificate verification for localhost it doesn't matter + #disable SSL certificate verification for localhost it doesn't matter await case.call_and_validate(verify=False) - \ No newline at end of file From d459a343a0abed2692afb95b14427b82a4ede789 Mon Sep 17 00:00:00 2001 From: Tarun Kumar Date: Wed, 16 Apr 2025 21:37:22 +0530 Subject: [PATCH 3/7] replace model Signed-off-by: Tarun Kumar --- tests/entrypoints/openai/test_openai_schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/entrypoints/openai/test_openai_schema.py b/tests/entrypoints/openai/test_openai_schema.py index d241dd0baf7..ffcd87483f1 100644 --- a/tests/entrypoints/openai/test_openai_schema.py +++ b/tests/entrypoints/openai/test_openai_schema.py @@ -7,7 +7,7 @@ schemathesis.experimental.OPEN_API_3_1.enable() -MODEL_NAME = "ibm-granite/granite-vision-3.2-2b" +MODEL_NAME = "HuggingFaceTB/SmolVLM-256M-Instruct" MAXIMUM_IMAGES = 2 From a2ceea67998039a1fd6a20b7599124697f795828 Mon Sep 17 00:00:00 2001 From: Tarun Kumar Date: Thu, 17 Apr 2025 15:49:14 +0530 Subject: [PATCH 4/7] apply header to all genereted tc from schema Signed-off-by: Tarun Kumar --- tests/entrypoints/openai/test_openai_schema.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/entrypoints/openai/test_openai_schema.py b/tests/entrypoints/openai/test_openai_schema.py index ffcd87483f1..1ccb803a328 100644 --- a/tests/entrypoints/openai/test_openai_schema.py +++ b/tests/entrypoints/openai/test_openai_schema.py @@ -43,9 +43,7 @@ def get_schema(server): @schema.parametrize() +@schema.override(headers={"Content-Type": "application/json"}) async def test_openapi_stateless(case): - case.headers = { - "Content-Type": "application/json", - } - #disable SSL certificate verification for localhost it doesn't matter + #No need to verify SSL certificate for localhost await case.call_and_validate(verify=False) From 9703dd5dcafad1cfde822e9f11a29a237cd0d92b Mon Sep 17 00:00:00 2001 From: Tarun Kumar Date: Thu, 17 Apr 2025 19:05:27 +0530 Subject: [PATCH 5/7] ignore Signed-off-by: Tarun Kumar --- .buildkite/test-pipeline.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml index 38961138c97..c9d15efc780 100644 --- a/.buildkite/test-pipeline.yaml +++ b/.buildkite/test-pipeline.yaml @@ -118,7 +118,7 @@ steps: - pytest -v -s entrypoints/llm/test_generate.py # it needs a clean process - pytest -v -s entrypoints/llm/test_generate_multiple_loras.py # it needs a clean process - VLLM_USE_V1=0 pytest -v -s entrypoints/llm/test_guided_generate.py # it needs a clean process - - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/test_oot_registration.py --ignore=entrypoints/openai/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/correctness/ + - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/test_oot_registration.py --ignore=entrypoints/openai/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/test_openai_schema.py - pytest -v -s entrypoints/test_chat_utils.py - VLLM_USE_V1=0 pytest -v -s entrypoints/offline_mode # Needs to avoid interference with other tests From 59ea0f303ba9687afd9466eee8509b722fab36ca Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Thu, 17 Apr 2025 07:45:24 -0700 Subject: [PATCH 6/7] [V1][Perf] Faster incremental detokenization (#15137) Signed-off-by: Nick Hill --- requirements/test.txt | 6 ------ 1 file changed, 6 deletions(-) diff --git a/requirements/test.txt b/requirements/test.txt index 174aa26a912..e7b8ec4e24b 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -688,12 +688,6 @@ tiktoken==0.7.0 # mistral-common timm==1.0.11 # via -r requirements/test.in -tokenizers==0.21.0 - # via transformers -tomli==2.2.1 - # via schemathesis -tomli-w==1.2.0 - # via schemathesis tokenizers==0.21.1 # via # -r requirements/test.in From 779db30f7fece8975ae3ead2a5134a8a76fd97df Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Thu, 17 Apr 2025 07:45:24 -0700 Subject: [PATCH 7/7] [V1][Perf] Faster incremental detokenization (#15137) Signed-off-by: Nick Hill --- requirements/test.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/requirements/test.txt b/requirements/test.txt index e7b8ec4e24b..6dcd4ff0146 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -692,6 +692,10 @@ tokenizers==0.21.1 # via # -r requirements/test.in # transformers +tomli==2.2.1 + # via schemathesis +tomli-w==1.2.0 + # via schemathesis torch==2.6.0 # via # -r requirements/test.in