|
| 1 | +# Copyright (c) The OGX Contributors. |
| 2 | +# All rights reserved. |
| 3 | +# |
| 4 | +# This source code is licensed under the terms described in the LICENSE file in |
| 5 | +# the root directory of this source tree. |
| 6 | + |
| 7 | +import mimetypes |
| 8 | + |
| 9 | +from fastapi import HTTPException, UploadFile |
| 10 | + |
| 11 | +from ogx.log import get_logger |
| 12 | +from ogx.providers.inline.file_processor.pypdf.config import PyPDFFileProcessorConfig |
| 13 | +from ogx.providers.inline.file_processor.pypdf.pypdf import PyPDFFileProcessor |
| 14 | +from ogx_api.file_processors import ProcessFileRequest, ProcessFileResponse |
| 15 | +from ogx_api.files import RetrieveFileRequest |
| 16 | + |
| 17 | +from .config import AutoFileProcessorConfig |
| 18 | + |
| 19 | +log = get_logger(name=__name__, category="providers::file_processors") |
| 20 | + |
| 21 | +SUPPORTED_TEXT_DESCRIPTION = "PDF and text files (txt, csv, md, etc.)" |
| 22 | + |
| 23 | + |
| 24 | +class AutoFileProcessor: |
| 25 | + """Composite file processor that dispatches to backends based on MIME type. |
| 26 | +
|
| 27 | + Routes PDF and text files to the built-in PyPDF processor. Unsupported |
| 28 | + formats are rejected with a 422 error listing the supported types. |
| 29 | + """ |
| 30 | + |
| 31 | + def __init__(self, config: AutoFileProcessorConfig, files_api) -> None: |
| 32 | + self.config = config |
| 33 | + self.files_api = files_api |
| 34 | + |
| 35 | + pypdf_config = PyPDFFileProcessorConfig( |
| 36 | + default_chunk_size_tokens=config.default_chunk_size_tokens, |
| 37 | + default_chunk_overlap_tokens=config.default_chunk_overlap_tokens, |
| 38 | + extract_metadata=config.extract_metadata, |
| 39 | + clean_text=config.clean_text, |
| 40 | + ) |
| 41 | + self.pypdf = PyPDFFileProcessor(pypdf_config, files_api) |
| 42 | + |
| 43 | + async def process_file( |
| 44 | + self, |
| 45 | + request: ProcessFileRequest, |
| 46 | + file: UploadFile | None = None, |
| 47 | + ) -> ProcessFileResponse: |
| 48 | + filename = await self._resolve_filename(request, file) |
| 49 | + mime_type, _ = mimetypes.guess_type(filename) |
| 50 | + mime_category = mime_type.split("/")[0] if (mime_type and "/" in mime_type) else None |
| 51 | + |
| 52 | + if mime_type == "application/pdf" or mime_category == "text": |
| 53 | + return await self.pypdf.process_file( |
| 54 | + file=file, |
| 55 | + file_id=request.file_id, |
| 56 | + options=request.options, |
| 57 | + chunking_strategy=request.chunking_strategy, |
| 58 | + ) |
| 59 | + |
| 60 | + raise HTTPException( |
| 61 | + status_code=422, |
| 62 | + detail=( |
| 63 | + f"File type '{mime_type or 'unknown'}' is not supported. Supported types: {SUPPORTED_TEXT_DESCRIPTION}." |
| 64 | + ), |
| 65 | + ) |
| 66 | + |
| 67 | + async def _resolve_filename(self, request: ProcessFileRequest, file: UploadFile | None) -> str: |
| 68 | + if file is not None: |
| 69 | + name: str | None = file.filename |
| 70 | + if name is not None: |
| 71 | + return name |
| 72 | + if request.file_id is not None: |
| 73 | + file_info = await self.files_api.openai_retrieve_file(RetrieveFileRequest(file_id=request.file_id)) |
| 74 | + resolved: str = file_info.filename |
| 75 | + return resolved |
| 76 | + return "unknown" |
| 77 | + |
| 78 | + async def shutdown(self) -> None: |
| 79 | + pass |
0 commit comments